udp: add batching to udp_rmem_release()
If udp_recvmsg() constantly releases sk_rmem_alloc for every read packet, it gives opportunity for producers to immediately grab spinlocks and desperatly try adding another packet, causing false sharing. We can add a simple heuristic to give the signal by batches of ~25 % of the queue capacity. This patch considerably increases performance under flood by about 50 %, since the thread draining the queue is no longer slowed by false sharing. Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
parent
c84d949057
commit
6b229cf77d
|
@ -79,6 +79,9 @@ struct udp_sock {
|
||||||
int (*gro_complete)(struct sock *sk,
|
int (*gro_complete)(struct sock *sk,
|
||||||
struct sk_buff *skb,
|
struct sk_buff *skb,
|
||||||
int nhoff);
|
int nhoff);
|
||||||
|
|
||||||
|
/* This field is dirtied by udp_recvmsg() */
|
||||||
|
int forward_deficit;
|
||||||
};
|
};
|
||||||
|
|
||||||
static inline struct udp_sock *udp_sk(const struct sock *sk)
|
static inline struct udp_sock *udp_sk(const struct sock *sk)
|
||||||
|
|
|
@ -1177,8 +1177,20 @@ out:
|
||||||
/* fully reclaim rmem/fwd memory allocated for skb */
|
/* fully reclaim rmem/fwd memory allocated for skb */
|
||||||
static void udp_rmem_release(struct sock *sk, int size, int partial)
|
static void udp_rmem_release(struct sock *sk, int size, int partial)
|
||||||
{
|
{
|
||||||
|
struct udp_sock *up = udp_sk(sk);
|
||||||
int amt;
|
int amt;
|
||||||
|
|
||||||
|
if (likely(partial)) {
|
||||||
|
up->forward_deficit += size;
|
||||||
|
size = up->forward_deficit;
|
||||||
|
if (size < (sk->sk_rcvbuf >> 2) &&
|
||||||
|
!skb_queue_empty(&sk->sk_receive_queue))
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
size += up->forward_deficit;
|
||||||
|
}
|
||||||
|
up->forward_deficit = 0;
|
||||||
|
|
||||||
atomic_sub(size, &sk->sk_rmem_alloc);
|
atomic_sub(size, &sk->sk_rmem_alloc);
|
||||||
sk->sk_forward_alloc += size;
|
sk->sk_forward_alloc += size;
|
||||||
amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1);
|
amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1);
|
||||||
|
|
Loading…
Reference in a new issue