b013840810
In PF_PACKET's packet mmap(), we can avoid using one atomic_inc() and one atomic_dec() call in skb destructor and use a percpu reference count instead in order to determine if packets are still pending to be sent out. Micro-benchmark with [1] that has been slightly modified (that is, protcol = 0 in socket(2) and bind(2)), example on a rather crappy testing machine; I expect it to scale and have even better results on bigger machines: ./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs: With patch: 4,022,015 cyc Without patch: 4,812,994 cyc time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable: With patch: real 1m32.241s user 0m0.287s sys 1m29.316s Without patch: real 1m38.386s user 0m0.265s sys 1m35.572s In function tpacket_snd(), it is okay to use packet_read_pending() since in fast-path we short-circuit the condition already with ph != NULL, since we have next frames to process. In case we have MSG_DONTWAIT, we also do not execute this path as need_wait is false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is okay to call a packet_read_pending(), because when we ever reach that path, we're done processing outgoing frames anyway and only look if there are skbs still outstanding to be orphaned. We can stay lockless in this percpu counter since it's acceptable when we reach this path for the sum to be imprecise first, but we'll level out at 0 after all pending frames have reached the skb destructor eventually through tx reclaim. When people pin a tx process to particular CPUs, we expect overflows to happen in the reference counter as on one CPU we expect heavy increase; and distributed through ksoftirqd on all CPUs a decrease, for example. As David Laight points out, since the C language doesn't define the result of signed int overflow (i.e. rather than wrap, it is allowed to saturate as a possible outcome), we have to use unsigned int as reference count. The sum over all CPUs when tx is complete will result in 0 again. The BUG_ON() in tpacket_destruct_skb() we can remove as well. It can _only_ be set from inside tpacket_snd() path and we made sure to increase tx_ring.pending in any case before we called po->xmit(skb). So testing for tx_ring.pending == 0 is not too useful. Instead, it would rather have been useful to test if lower layers didn't orphan the skb so that we're missing ring slots being put back to TP_STATUS_AVAILABLE. But such a bug will be caught in user space already as we end up realizing that we do not have any TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set. Btw, in case of RX_RING path, we do not make use of the pending member, therefore we also don't need to use up any percpu memory here. Also note that __alloc_percpu() already returns a zero-filled percpu area, so initialization is done already. [1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap Signed-off-by: Daniel Borkmann <dborkman@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
127 lines
2.8 KiB
C
127 lines
2.8 KiB
C
#ifndef __PACKET_INTERNAL_H__
|
|
#define __PACKET_INTERNAL_H__
|
|
|
|
struct packet_mclist {
|
|
struct packet_mclist *next;
|
|
int ifindex;
|
|
int count;
|
|
unsigned short type;
|
|
unsigned short alen;
|
|
unsigned char addr[MAX_ADDR_LEN];
|
|
};
|
|
|
|
/* kbdq - kernel block descriptor queue */
|
|
struct tpacket_kbdq_core {
|
|
struct pgv *pkbdq;
|
|
unsigned int feature_req_word;
|
|
unsigned int hdrlen;
|
|
unsigned char reset_pending_on_curr_blk;
|
|
unsigned char delete_blk_timer;
|
|
unsigned short kactive_blk_num;
|
|
unsigned short blk_sizeof_priv;
|
|
|
|
/* last_kactive_blk_num:
|
|
* trick to see if user-space has caught up
|
|
* in order to avoid refreshing timer when every single pkt arrives.
|
|
*/
|
|
unsigned short last_kactive_blk_num;
|
|
|
|
char *pkblk_start;
|
|
char *pkblk_end;
|
|
int kblk_size;
|
|
unsigned int knum_blocks;
|
|
uint64_t knxt_seq_num;
|
|
char *prev;
|
|
char *nxt_offset;
|
|
struct sk_buff *skb;
|
|
|
|
atomic_t blk_fill_in_prog;
|
|
|
|
/* Default is set to 8ms */
|
|
#define DEFAULT_PRB_RETIRE_TOV (8)
|
|
|
|
unsigned short retire_blk_tov;
|
|
unsigned short version;
|
|
unsigned long tov_in_jiffies;
|
|
|
|
/* timer to retire an outstanding block */
|
|
struct timer_list retire_blk_timer;
|
|
};
|
|
|
|
struct pgv {
|
|
char *buffer;
|
|
};
|
|
|
|
struct packet_ring_buffer {
|
|
struct pgv *pg_vec;
|
|
|
|
unsigned int head;
|
|
unsigned int frames_per_block;
|
|
unsigned int frame_size;
|
|
unsigned int frame_max;
|
|
|
|
unsigned int pg_vec_order;
|
|
unsigned int pg_vec_pages;
|
|
unsigned int pg_vec_len;
|
|
|
|
unsigned int __percpu *pending_refcnt;
|
|
|
|
struct tpacket_kbdq_core prb_bdqc;
|
|
};
|
|
|
|
extern struct mutex fanout_mutex;
|
|
#define PACKET_FANOUT_MAX 256
|
|
|
|
struct packet_fanout {
|
|
#ifdef CONFIG_NET_NS
|
|
struct net *net;
|
|
#endif
|
|
unsigned int num_members;
|
|
u16 id;
|
|
u8 type;
|
|
u8 flags;
|
|
atomic_t rr_cur;
|
|
struct list_head list;
|
|
struct sock *arr[PACKET_FANOUT_MAX];
|
|
int next[PACKET_FANOUT_MAX];
|
|
spinlock_t lock;
|
|
atomic_t sk_ref;
|
|
struct packet_type prot_hook ____cacheline_aligned_in_smp;
|
|
};
|
|
|
|
struct packet_sock {
|
|
/* struct sock has to be the first member of packet_sock */
|
|
struct sock sk;
|
|
struct packet_fanout *fanout;
|
|
union tpacket_stats_u stats;
|
|
struct packet_ring_buffer rx_ring;
|
|
struct packet_ring_buffer tx_ring;
|
|
int copy_thresh;
|
|
spinlock_t bind_lock;
|
|
struct mutex pg_vec_lock;
|
|
unsigned int running:1, /* prot_hook is attached*/
|
|
auxdata:1,
|
|
origdev:1,
|
|
has_vnet_hdr:1;
|
|
int ifindex; /* bound device */
|
|
__be16 num;
|
|
struct packet_mclist *mclist;
|
|
atomic_t mapped;
|
|
enum tpacket_versions tp_version;
|
|
unsigned int tp_hdrlen;
|
|
unsigned int tp_reserve;
|
|
unsigned int tp_loss:1;
|
|
unsigned int tp_tx_has_off:1;
|
|
unsigned int tp_tstamp;
|
|
struct net_device __rcu *cached_dev;
|
|
int (*xmit)(struct sk_buff *skb);
|
|
struct packet_type prot_hook ____cacheline_aligned_in_smp;
|
|
};
|
|
|
|
static struct packet_sock *pkt_sk(struct sock *sk)
|
|
{
|
|
return (struct packet_sock *)sk;
|
|
}
|
|
|
|
#endif
|