![Alexei Starovoitov](/assets/img/avatar_default.png)
Introduce 'struct bpf_spin_lock' and bpf_spin_lock/unlock() helpers to let bpf program serialize access to other variables. Example: struct hash_elem { int cnt; struct bpf_spin_lock lock; }; struct hash_elem * val = bpf_map_lookup_elem(&hash_map, &key); if (val) { bpf_spin_lock(&val->lock); val->cnt++; bpf_spin_unlock(&val->lock); } Restrictions and safety checks: - bpf_spin_lock is only allowed inside HASH and ARRAY maps. - BTF description of the map is mandatory for safety analysis. - bpf program can take one bpf_spin_lock at a time, since two or more can cause dead locks. - only one 'struct bpf_spin_lock' is allowed per map element. It drastically simplifies implementation yet allows bpf program to use any number of bpf_spin_locks. - when bpf_spin_lock is taken the calls (either bpf2bpf or helpers) are not allowed. - bpf program must bpf_spin_unlock() before return. - bpf program can access 'struct bpf_spin_lock' only via bpf_spin_lock()/bpf_spin_unlock() helpers. - load/store into 'struct bpf_spin_lock lock;' field is not allowed. - to use bpf_spin_lock() helper the BTF description of map value must be a struct and have 'struct bpf_spin_lock anyname;' field at the top level. Nested lock inside another struct is not allowed. - syscall map_lookup doesn't copy bpf_spin_lock field to user space. - syscall map_update and program map_update do not update bpf_spin_lock field. - bpf_spin_lock cannot be on the stack or inside networking packet. bpf_spin_lock can only be inside HASH or ARRAY map value. - bpf_spin_lock is available to root only and to all program types. - bpf_spin_lock is not allowed in inner maps of map-in-map. - ld_abs is not allowed inside spin_lock-ed region. - tracing progs and socket filter progs cannot use bpf_spin_lock due to insufficient preemption checks Implementation details: - cgroup-bpf class of programs can nest with xdp/tc programs. Hence bpf_spin_lock is equivalent to spin_lock_irqsave. Other solutions to avoid nested bpf_spin_lock are possible. Like making sure that all networking progs run with softirq disabled. spin_lock_irqsave is the simplest and doesn't add overhead to the programs that don't use it. - arch_spinlock_t is used when its implemented as queued_spin_lock - archs can force their own arch_spinlock_t - on architectures where queued_spin_lock is not available and sizeof(arch_spinlock_t) != sizeof(__u32) trivial lock is used. - presence of bpf_spin_lock inside map value could have been indicated via extra flag during map_create, but specifying it via BTF is cleaner. It provides introspection for map key/value and reduces user mistakes. Next steps: - allow bpf_spin_lock in other map types (like cgroup local storage) - introduce BPF_F_LOCK flag for bpf_map_update() syscall and helper to request kernel to grab bpf_spin_lock before rewriting the value. That will serialize access to map elements. Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
254 lines
4.8 KiB
Plaintext
254 lines
4.8 KiB
Plaintext
#
|
|
# The ARCH_INLINE foo is necessary because select ignores "depends on"
|
|
#
|
|
config ARCH_INLINE_SPIN_TRYLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_TRYLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK_IRQSAVE
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE
|
|
bool
|
|
|
|
|
|
config ARCH_INLINE_READ_TRYLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK_IRQSAVE
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK_IRQRESTORE
|
|
bool
|
|
|
|
|
|
config ARCH_INLINE_WRITE_TRYLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK_IRQSAVE
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
|
|
bool
|
|
|
|
config UNINLINE_SPIN_UNLOCK
|
|
bool
|
|
|
|
#
|
|
# lock_* functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and GENERIC_LOCKBREAK=n and ARCH_INLINE_*LOCK=y
|
|
#
|
|
# trylock_* functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y
|
|
#
|
|
# unlock and unlock_irq functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y
|
|
# or
|
|
# - DEBUG_SPINLOCK=n and PREEMPT=n
|
|
#
|
|
# unlock_bh and unlock_irqrestore functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y
|
|
#
|
|
|
|
if !DEBUG_SPINLOCK
|
|
|
|
config INLINE_SPIN_TRYLOCK
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_TRYLOCK
|
|
|
|
config INLINE_SPIN_TRYLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_TRYLOCK_BH
|
|
|
|
config INLINE_SPIN_LOCK
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK
|
|
|
|
config INLINE_SPIN_LOCK_BH
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK_BH
|
|
|
|
config INLINE_SPIN_LOCK_IRQ
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK_IRQ
|
|
|
|
config INLINE_SPIN_LOCK_IRQSAVE
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK_IRQSAVE
|
|
|
|
config INLINE_SPIN_UNLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_UNLOCK_BH
|
|
|
|
config INLINE_SPIN_UNLOCK_IRQ
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_SPIN_UNLOCK_IRQ
|
|
|
|
config INLINE_SPIN_UNLOCK_IRQRESTORE
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE
|
|
|
|
|
|
config INLINE_READ_TRYLOCK
|
|
def_bool y
|
|
depends on ARCH_INLINE_READ_TRYLOCK
|
|
|
|
config INLINE_READ_LOCK
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK
|
|
|
|
config INLINE_READ_LOCK_BH
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK_BH
|
|
|
|
config INLINE_READ_LOCK_IRQ
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK_IRQ
|
|
|
|
config INLINE_READ_LOCK_IRQSAVE
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK_IRQSAVE
|
|
|
|
config INLINE_READ_UNLOCK
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_READ_UNLOCK
|
|
|
|
config INLINE_READ_UNLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_READ_UNLOCK_BH
|
|
|
|
config INLINE_READ_UNLOCK_IRQ
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_READ_UNLOCK_IRQ
|
|
|
|
config INLINE_READ_UNLOCK_IRQRESTORE
|
|
def_bool y
|
|
depends on ARCH_INLINE_READ_UNLOCK_IRQRESTORE
|
|
|
|
|
|
config INLINE_WRITE_TRYLOCK
|
|
def_bool y
|
|
depends on ARCH_INLINE_WRITE_TRYLOCK
|
|
|
|
config INLINE_WRITE_LOCK
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK
|
|
|
|
config INLINE_WRITE_LOCK_BH
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK_BH
|
|
|
|
config INLINE_WRITE_LOCK_IRQ
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK_IRQ
|
|
|
|
config INLINE_WRITE_LOCK_IRQSAVE
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK_IRQSAVE
|
|
|
|
config INLINE_WRITE_UNLOCK
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_WRITE_UNLOCK
|
|
|
|
config INLINE_WRITE_UNLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_WRITE_UNLOCK_BH
|
|
|
|
config INLINE_WRITE_UNLOCK_IRQ
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_WRITE_UNLOCK_IRQ
|
|
|
|
config INLINE_WRITE_UNLOCK_IRQRESTORE
|
|
def_bool y
|
|
depends on ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
|
|
|
|
endif
|
|
|
|
config ARCH_SUPPORTS_ATOMIC_RMW
|
|
bool
|
|
|
|
config MUTEX_SPIN_ON_OWNER
|
|
def_bool y
|
|
depends on SMP && ARCH_SUPPORTS_ATOMIC_RMW
|
|
|
|
config RWSEM_SPIN_ON_OWNER
|
|
def_bool y
|
|
depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW
|
|
|
|
config LOCK_SPIN_ON_OWNER
|
|
def_bool y
|
|
depends on MUTEX_SPIN_ON_OWNER || RWSEM_SPIN_ON_OWNER
|
|
|
|
config ARCH_USE_QUEUED_SPINLOCKS
|
|
bool
|
|
|
|
config QUEUED_SPINLOCKS
|
|
def_bool y if ARCH_USE_QUEUED_SPINLOCKS
|
|
depends on SMP
|
|
|
|
config BPF_ARCH_SPINLOCK
|
|
bool
|
|
|
|
config ARCH_USE_QUEUED_RWLOCKS
|
|
bool
|
|
|
|
config QUEUED_RWLOCKS
|
|
def_bool y if ARCH_USE_QUEUED_RWLOCKS
|
|
depends on SMP
|