![Willem de Bruijn](/assets/img/avatar_default.png)
UDP reuseport groups can hold a mix unconnected and connected sockets.
Ensure that connections only receive all traffic to their 4-tuple.
Fast reuseport returns on the first reuseport match on the assumption
that all matches are equal. Only if connections are present, return to
the previous behavior of scoring all sockets.
Record if connections are present and if so (1) treat such connected
sockets as an independent match from the group, (2) only return
2-tuple matches from reuseport and (3) do not return on the first
2-tuple reuseport match to allow for a higher scoring match later.
New field has_conns is set without locks. No other fields in the
bitmap are modified at runtime and the field is only ever set
unconditionally, so an RMW cannot miss a change.
Fixes: e32ea7e747
("soreuseport: fast reuseport UDP socket selection")
Link: http://lkml.kernel.org/r/CA+FuTSfRP09aJNYRt04SS6qj22ViiOEWaWmLAwX0psk8-PGNxw@mail.gmail.com
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Craig Gallek <kraig@google.com>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
370 lines
9.3 KiB
C
370 lines
9.3 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* To speed up listener socket lookup, create an array to store all sockets
|
|
* listening on the same port. This allows a decision to be made after finding
|
|
* the first socket. An optional BPF program can also be configured for
|
|
* selecting the socket index from the array of available sockets.
|
|
*/
|
|
|
|
#include <net/sock_reuseport.h>
|
|
#include <linux/bpf.h>
|
|
#include <linux/idr.h>
|
|
#include <linux/filter.h>
|
|
#include <linux/rcupdate.h>
|
|
|
|
#define INIT_SOCKS 128
|
|
|
|
DEFINE_SPINLOCK(reuseport_lock);
|
|
|
|
#define REUSEPORT_MIN_ID 1
|
|
static DEFINE_IDA(reuseport_ida);
|
|
|
|
int reuseport_get_id(struct sock_reuseport *reuse)
|
|
{
|
|
int id;
|
|
|
|
if (reuse->reuseport_id)
|
|
return reuse->reuseport_id;
|
|
|
|
id = ida_simple_get(&reuseport_ida, REUSEPORT_MIN_ID, 0,
|
|
/* Called under reuseport_lock */
|
|
GFP_ATOMIC);
|
|
if (id < 0)
|
|
return id;
|
|
|
|
reuse->reuseport_id = id;
|
|
|
|
return reuse->reuseport_id;
|
|
}
|
|
|
|
static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks)
|
|
{
|
|
unsigned int size = sizeof(struct sock_reuseport) +
|
|
sizeof(struct sock *) * max_socks;
|
|
struct sock_reuseport *reuse = kzalloc(size, GFP_ATOMIC);
|
|
|
|
if (!reuse)
|
|
return NULL;
|
|
|
|
reuse->max_socks = max_socks;
|
|
|
|
RCU_INIT_POINTER(reuse->prog, NULL);
|
|
return reuse;
|
|
}
|
|
|
|
int reuseport_alloc(struct sock *sk, bool bind_inany)
|
|
{
|
|
struct sock_reuseport *reuse;
|
|
|
|
/* bh lock used since this function call may precede hlist lock in
|
|
* soft irq of receive path or setsockopt from process context
|
|
*/
|
|
spin_lock_bh(&reuseport_lock);
|
|
|
|
/* Allocation attempts can occur concurrently via the setsockopt path
|
|
* and the bind/hash path. Nothing to do when we lose the race.
|
|
*/
|
|
reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
|
|
lockdep_is_held(&reuseport_lock));
|
|
if (reuse) {
|
|
/* Only set reuse->bind_inany if the bind_inany is true.
|
|
* Otherwise, it will overwrite the reuse->bind_inany
|
|
* which was set by the bind/hash path.
|
|
*/
|
|
if (bind_inany)
|
|
reuse->bind_inany = bind_inany;
|
|
goto out;
|
|
}
|
|
|
|
reuse = __reuseport_alloc(INIT_SOCKS);
|
|
if (!reuse) {
|
|
spin_unlock_bh(&reuseport_lock);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
reuse->socks[0] = sk;
|
|
reuse->num_socks = 1;
|
|
reuse->bind_inany = bind_inany;
|
|
rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
|
|
|
|
out:
|
|
spin_unlock_bh(&reuseport_lock);
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(reuseport_alloc);
|
|
|
|
static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)
|
|
{
|
|
struct sock_reuseport *more_reuse;
|
|
u32 more_socks_size, i;
|
|
|
|
more_socks_size = reuse->max_socks * 2U;
|
|
if (more_socks_size > U16_MAX)
|
|
return NULL;
|
|
|
|
more_reuse = __reuseport_alloc(more_socks_size);
|
|
if (!more_reuse)
|
|
return NULL;
|
|
|
|
more_reuse->max_socks = more_socks_size;
|
|
more_reuse->num_socks = reuse->num_socks;
|
|
more_reuse->prog = reuse->prog;
|
|
more_reuse->reuseport_id = reuse->reuseport_id;
|
|
more_reuse->bind_inany = reuse->bind_inany;
|
|
|
|
memcpy(more_reuse->socks, reuse->socks,
|
|
reuse->num_socks * sizeof(struct sock *));
|
|
more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts);
|
|
|
|
for (i = 0; i < reuse->num_socks; ++i)
|
|
rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb,
|
|
more_reuse);
|
|
|
|
/* Note: we use kfree_rcu here instead of reuseport_free_rcu so
|
|
* that reuse and more_reuse can temporarily share a reference
|
|
* to prog.
|
|
*/
|
|
kfree_rcu(reuse, rcu);
|
|
return more_reuse;
|
|
}
|
|
|
|
static void reuseport_free_rcu(struct rcu_head *head)
|
|
{
|
|
struct sock_reuseport *reuse;
|
|
|
|
reuse = container_of(head, struct sock_reuseport, rcu);
|
|
sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1));
|
|
if (reuse->reuseport_id)
|
|
ida_simple_remove(&reuseport_ida, reuse->reuseport_id);
|
|
kfree(reuse);
|
|
}
|
|
|
|
/**
|
|
* reuseport_add_sock - Add a socket to the reuseport group of another.
|
|
* @sk: New socket to add to the group.
|
|
* @sk2: Socket belonging to the existing reuseport group.
|
|
* @bind_inany: Whether or not the group is bound to a local INANY address.
|
|
*
|
|
* May return ENOMEM and not add socket to group under memory pressure.
|
|
*/
|
|
int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany)
|
|
{
|
|
struct sock_reuseport *old_reuse, *reuse;
|
|
|
|
if (!rcu_access_pointer(sk2->sk_reuseport_cb)) {
|
|
int err = reuseport_alloc(sk2, bind_inany);
|
|
|
|
if (err)
|
|
return err;
|
|
}
|
|
|
|
spin_lock_bh(&reuseport_lock);
|
|
reuse = rcu_dereference_protected(sk2->sk_reuseport_cb,
|
|
lockdep_is_held(&reuseport_lock));
|
|
old_reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
|
|
lockdep_is_held(&reuseport_lock));
|
|
if (old_reuse && old_reuse->num_socks != 1) {
|
|
spin_unlock_bh(&reuseport_lock);
|
|
return -EBUSY;
|
|
}
|
|
|
|
if (reuse->num_socks == reuse->max_socks) {
|
|
reuse = reuseport_grow(reuse);
|
|
if (!reuse) {
|
|
spin_unlock_bh(&reuseport_lock);
|
|
return -ENOMEM;
|
|
}
|
|
}
|
|
|
|
reuse->socks[reuse->num_socks] = sk;
|
|
/* paired with smp_rmb() in reuseport_select_sock() */
|
|
smp_wmb();
|
|
reuse->num_socks++;
|
|
rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
|
|
|
|
spin_unlock_bh(&reuseport_lock);
|
|
|
|
if (old_reuse)
|
|
call_rcu(&old_reuse->rcu, reuseport_free_rcu);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(reuseport_add_sock);
|
|
|
|
void reuseport_detach_sock(struct sock *sk)
|
|
{
|
|
struct sock_reuseport *reuse;
|
|
int i;
|
|
|
|
spin_lock_bh(&reuseport_lock);
|
|
reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
|
|
lockdep_is_held(&reuseport_lock));
|
|
|
|
/* At least one of the sk in this reuseport group is added to
|
|
* a bpf map. Notify the bpf side. The bpf map logic will
|
|
* remove the sk if it is indeed added to a bpf map.
|
|
*/
|
|
if (reuse->reuseport_id)
|
|
bpf_sk_reuseport_detach(sk);
|
|
|
|
rcu_assign_pointer(sk->sk_reuseport_cb, NULL);
|
|
|
|
for (i = 0; i < reuse->num_socks; i++) {
|
|
if (reuse->socks[i] == sk) {
|
|
reuse->socks[i] = reuse->socks[reuse->num_socks - 1];
|
|
reuse->num_socks--;
|
|
if (reuse->num_socks == 0)
|
|
call_rcu(&reuse->rcu, reuseport_free_rcu);
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock_bh(&reuseport_lock);
|
|
}
|
|
EXPORT_SYMBOL(reuseport_detach_sock);
|
|
|
|
static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,
|
|
struct bpf_prog *prog, struct sk_buff *skb,
|
|
int hdr_len)
|
|
{
|
|
struct sk_buff *nskb = NULL;
|
|
u32 index;
|
|
|
|
if (skb_shared(skb)) {
|
|
nskb = skb_clone(skb, GFP_ATOMIC);
|
|
if (!nskb)
|
|
return NULL;
|
|
skb = nskb;
|
|
}
|
|
|
|
/* temporarily advance data past protocol header */
|
|
if (!pskb_pull(skb, hdr_len)) {
|
|
kfree_skb(nskb);
|
|
return NULL;
|
|
}
|
|
index = bpf_prog_run_save_cb(prog, skb);
|
|
__skb_push(skb, hdr_len);
|
|
|
|
consume_skb(nskb);
|
|
|
|
if (index >= socks)
|
|
return NULL;
|
|
|
|
return reuse->socks[index];
|
|
}
|
|
|
|
/**
|
|
* reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
|
|
* @sk: First socket in the group.
|
|
* @hash: When no BPF filter is available, use this hash to select.
|
|
* @skb: skb to run through BPF filter.
|
|
* @hdr_len: BPF filter expects skb data pointer at payload data. If
|
|
* the skb does not yet point at the payload, this parameter represents
|
|
* how far the pointer needs to advance to reach the payload.
|
|
* Returns a socket that should receive the packet (or NULL on error).
|
|
*/
|
|
struct sock *reuseport_select_sock(struct sock *sk,
|
|
u32 hash,
|
|
struct sk_buff *skb,
|
|
int hdr_len)
|
|
{
|
|
struct sock_reuseport *reuse;
|
|
struct bpf_prog *prog;
|
|
struct sock *sk2 = NULL;
|
|
u16 socks;
|
|
|
|
rcu_read_lock();
|
|
reuse = rcu_dereference(sk->sk_reuseport_cb);
|
|
|
|
/* if memory allocation failed or add call is not yet complete */
|
|
if (!reuse)
|
|
goto out;
|
|
|
|
prog = rcu_dereference(reuse->prog);
|
|
socks = READ_ONCE(reuse->num_socks);
|
|
if (likely(socks)) {
|
|
/* paired with smp_wmb() in reuseport_add_sock() */
|
|
smp_rmb();
|
|
|
|
if (!prog || !skb)
|
|
goto select_by_hash;
|
|
|
|
if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
|
|
sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, hash);
|
|
else
|
|
sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len);
|
|
|
|
select_by_hash:
|
|
/* no bpf or invalid bpf result: fall back to hash usage */
|
|
if (!sk2) {
|
|
int i, j;
|
|
|
|
i = j = reciprocal_scale(hash, socks);
|
|
while (reuse->socks[i]->sk_state == TCP_ESTABLISHED) {
|
|
i++;
|
|
if (i >= reuse->num_socks)
|
|
i = 0;
|
|
if (i == j)
|
|
goto out;
|
|
}
|
|
sk2 = reuse->socks[i];
|
|
}
|
|
}
|
|
|
|
out:
|
|
rcu_read_unlock();
|
|
return sk2;
|
|
}
|
|
EXPORT_SYMBOL(reuseport_select_sock);
|
|
|
|
int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog)
|
|
{
|
|
struct sock_reuseport *reuse;
|
|
struct bpf_prog *old_prog;
|
|
|
|
if (sk_unhashed(sk) && sk->sk_reuseport) {
|
|
int err = reuseport_alloc(sk, false);
|
|
|
|
if (err)
|
|
return err;
|
|
} else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
|
|
/* The socket wasn't bound with SO_REUSEPORT */
|
|
return -EINVAL;
|
|
}
|
|
|
|
spin_lock_bh(&reuseport_lock);
|
|
reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
|
|
lockdep_is_held(&reuseport_lock));
|
|
old_prog = rcu_dereference_protected(reuse->prog,
|
|
lockdep_is_held(&reuseport_lock));
|
|
rcu_assign_pointer(reuse->prog, prog);
|
|
spin_unlock_bh(&reuseport_lock);
|
|
|
|
sk_reuseport_prog_free(old_prog);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(reuseport_attach_prog);
|
|
|
|
int reuseport_detach_prog(struct sock *sk)
|
|
{
|
|
struct sock_reuseport *reuse;
|
|
struct bpf_prog *old_prog;
|
|
|
|
if (!rcu_access_pointer(sk->sk_reuseport_cb))
|
|
return sk->sk_reuseport ? -ENOENT : -EINVAL;
|
|
|
|
old_prog = NULL;
|
|
spin_lock_bh(&reuseport_lock);
|
|
reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
|
|
lockdep_is_held(&reuseport_lock));
|
|
rcu_swap_protected(reuse->prog, old_prog,
|
|
lockdep_is_held(&reuseport_lock));
|
|
spin_unlock_bh(&reuseport_lock);
|
|
|
|
if (!old_prog)
|
|
return -ENOENT;
|
|
|
|
sk_reuseport_prog_free(old_prog);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(reuseport_detach_prog);
|