redonkable
/
alistair23-linux
1
0
Fork 0
Code Releases Activity

spinlock: lockbreak cleanup 

The break_lock data structure and code for spinlocks is quite nasty.
Not only does it double the size of a spinlock but it changes locking to
a potentially less optimal trylock.

Put all of that under CONFIG_GENERIC_LOCKBREAK, and introduce a
__raw_spin_is_contended that uses the lock data itself to determine whether
there are waiters on the lock, to be used if CONFIG_GENERIC_LOCKBREAK is
not set.

Rename need_lockbreak to spin_needbreak, make it use spin_is_contended to
decouple it from the spinlock implementation, and make it typesafe (rwlocks
do not have any need_lockbreak sites -- why do they even get bloated up
with that break_lock then?).

Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
hifive-unleashed-5.1
Nick Piggin 2008-01-30 13:31:20 +01:00 committed by Ingo Molnar
parent a95d67f87e
commit 95c354fe9f
19 changed files with 72 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
arch/arm/Kconfig
View File

@ -91,6 +91,11 @@ config GENERIC_IRQ_PROBE
bool
default y
config GENERIC_LOCKBREAK
bool
default y
depends on SMP && PREEMPT
config RWSEM_GENERIC_SPINLOCK
bool
default y

5
arch/ia64/Kconfig
View File

@ -42,6 +42,11 @@ config MMU
config SWIOTLB
bool
config GENERIC_LOCKBREAK
bool
default y
depends on SMP && PREEMPT
config RWSEM_XCHGADD_ALGORITHM
bool
default y

5
arch/m32r/Kconfig
View File

@ -235,6 +235,11 @@ config IRAM_SIZE
# Define implied options from the CPU selection here
#
config GENERIC_LOCKBREAK
bool
default y
depends on SMP && PREEMPT
config RWSEM_GENERIC_SPINLOCK
bool
depends on M32R

5
arch/mips/Kconfig
View File

@ -694,6 +694,11 @@ source "arch/mips/vr41xx/Kconfig"
endmenu
config GENERIC_LOCKBREAK
bool
default y
depends on SMP && PREEMPT
config RWSEM_GENERIC_SPINLOCK
bool
default y

5
arch/parisc/Kconfig
View File

@ -19,6 +19,11 @@ config MMU
config STACK_GROWSUP
def_bool y
config GENERIC_LOCKBREAK
bool
default y
depends on SMP && PREEMPT
config RWSEM_GENERIC_SPINLOCK
def_bool y

5
arch/powerpc/Kconfig
View File

@ -53,6 +53,11 @@ config RWSEM_XCHGADD_ALGORITHM
bool
default y
config GENERIC_LOCKBREAK
bool
default y
depends on SMP && PREEMPT
config ARCH_HAS_ILOG2_U32
bool
default y

5
arch/sparc64/Kconfig
View File

@ -200,6 +200,11 @@ config US2E_FREQ
If in doubt, say N.
# Global things across all Sun machines.
config GENERIC_LOCKBREAK
bool
default y
depends on SMP && PREEMPT
config RWSEM_GENERIC_SPINLOCK
bool

4
arch/x86/Kconfig
View File

@ -19,6 +19,10 @@ config X86_64
config X86
def_bool y
config GENERIC_LOCKBREAK
def_bool y
depends on SMP && PREEMPT
config GENERIC_TIME
def_bool y

3
fs/jbd/checkpoint.c
View File

@ -347,7 +347,8 @@ restart:
break;
}
retry = __process_buffer(journal, jh, bhs,&batch_count);
if (!retry && lock_need_resched(&journal->j_list_lock)){
if (!retry && (need_resched() ||
spin_needbreak(&journal->j_list_lock))) {
spin_unlock(&journal->j_list_lock);
retry = 1;
break;

2
fs/jbd/commit.c
View File

@ -265,7 +265,7 @@ write_out_data:
put_bh(bh);
}
if (lock_need_resched(&journal->j_list_lock)) {
if (need_resched() || spin_needbreak(&journal->j_list_lock)) {
spin_unlock(&journal->j_list_lock);
goto write_out_data;
}

3
fs/jbd2/checkpoint.c
View File

@ -353,7 +353,8 @@ restart:
}
retry = __process_buffer(journal, jh, bhs, &batch_count,
transaction);
if (!retry && lock_need_resched(&journal->j_list_lock)){
if (!retry && (need_resched() ||
spin_needbreak(&journal->j_list_lock))) {
spin_unlock(&journal->j_list_lock);
retry = 1;
break;

2
fs/jbd2/commit.c
View File

@ -341,7 +341,7 @@ write_out_data:
put_bh(bh);
}
if (lock_need_resched(&journal->j_list_lock)) {
if (need_resched() || spin_needbreak(&journal->j_list_lock)) {
spin_unlock(&journal->j_list_lock);
goto write_out_data;
}

21
include/linux/sched.h
View File

@ -1922,23 +1922,16 @@ extern int cond_resched_softirq(void);
/*
* Does a critical section need to be broken due to another
* task waiting?:
* task waiting?: (technically does not depend on CONFIG_PREEMPT,
* but a general need for low latency)
*/
#if defined(CONFIG_PREEMPT) && defined(CONFIG_SMP)
# define need_lockbreak(lock) ((lock)->break_lock)
#else
# define need_lockbreak(lock) 0
#endif
/*
* Does a critical section need to be broken due to another
* task waiting or preemption being signalled:
*/
static inline int lock_need_resched(spinlock_t *lock)
static inline int spin_needbreak(spinlock_t *lock)
{
if (need_lockbreak(lock) || need_resched())
return 1;
#ifdef CONFIG_PREEMPT
return spin_is_contended(lock);
#else
return 0;
#endif
}
/*

6
include/linux/spinlock.h
View File

@ -120,6 +120,12 @@ do { \
#define spin_is_locked(lock) __raw_spin_is_locked(&(lock)->raw_lock)
#ifdef CONFIG_GENERIC_LOCKBREAK
#define spin_is_contended(lock) ((lock)->break_lock)
#else
#define spin_is_contended(lock) __raw_spin_is_contended(&(lock)->raw_lock)
#endif
/**
* spin_unlock_wait - wait until the spinlock gets unlocked
* @lock: the spinlock in question.

4
include/linux/spinlock_types.h
View File

@ -19,7 +19,7 @@
typedef struct {
raw_spinlock_t raw_lock;
#if defined(CONFIG_PREEMPT) && defined(CONFIG_SMP)
#ifdef CONFIG_GENERIC_LOCKBREAK
unsigned int break_lock;
#endif
#ifdef CONFIG_DEBUG_SPINLOCK
@ -35,7 +35,7 @@ typedef struct {
typedef struct {
raw_rwlock_t raw_lock;
#if defined(CONFIG_PREEMPT) && defined(CONFIG_SMP)
#ifdef CONFIG_GENERIC_LOCKBREAK
unsigned int break_lock;
#endif
#ifdef CONFIG_DEBUG_SPINLOCK

2
include/linux/spinlock_up.h
View File

@ -64,6 +64,8 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock)
# define __raw_spin_trylock(lock) ({ (void)(lock); 1; })
#endif /* DEBUG_SPINLOCK */
#define __raw_spin_is_contended(lock) (((void)(lock), 0))
#define __raw_read_can_lock(lock) (((void)(lock), 1))
#define __raw_write_can_lock(lock) (((void)(lock), 1))

16
kernel/sched.c
View File

@ -4945,19 +4945,15 @@ EXPORT_SYMBOL(_cond_resched);
*/
int cond_resched_lock(spinlock_t *lock)
{
int resched = need_resched() && system_state == SYSTEM_RUNNING;
int ret = 0;
if (need_lockbreak(lock)) {
if (spin_needbreak(lock) || resched) {
spin_unlock(lock);
cpu_relax();
ret = 1;
spin_lock(lock);
}
if (need_resched() && system_state == SYSTEM_RUNNING) {
spin_release(&lock->dep_map, 1, _THIS_IP_);
_raw_spin_unlock(lock);
preempt_enable_no_resched();
__cond_resched();
if (resched && need_resched())
__cond_resched();
else
cpu_relax();
ret = 1;
spin_lock(lock);
}

3
kernel/spinlock.c
View File

@ -65,8 +65,7 @@ EXPORT_SYMBOL(_write_trylock);
* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
* not re-enabled during lock-acquire (which the preempt-spin-ops do):
*/
#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
defined(CONFIG_DEBUG_LOCK_ALLOC)
#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
void __lockfunc _read_lock(rwlock_t *lock)
{

8
mm/memory.c
View File

@ -513,8 +513,7 @@ again:
if (progress >= 32) {
progress = 0;
if (need_resched() ||
need_lockbreak(src_ptl) ||
need_lockbreak(dst_ptl))
spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
break;
}
if (pte_none(*src_pte)) {
@ -853,7 +852,7 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
tlb_finish_mmu(*tlbp, tlb_start, start);
if (need_resched() ||
(i_mmap_lock && need_lockbreak(i_mmap_lock))) {
(i_mmap_lock && spin_needbreak(i_mmap_lock))) {
if (i_mmap_lock) {
*tlbp = NULL;
goto out;
@ -1768,8 +1767,7 @@ again:
restart_addr = zap_page_range(vma, start_addr,
end_addr - start_addr, details);
need_break = need_resched() ||
need_lockbreak(details->i_mmap_lock);
need_break = need_resched() || spin_needbreak(details->i_mmap_lock);
if (restart_addr >= end_addr) {
/* We have now completed this vma: mark it so */