net: release dst entry in dev_hard_start_xmit()
One point of contention in high network loads is the dst_release() performed when a transmited skb is freed. This is because NIC tx completion calls dev_kree_skb() long after original call to dev_queue_xmit(skb). CPU cache is cold and the atomic op in dst_release() stalls. On SMP, this is quite visible if one CPU is 100% handling softirqs for a network device, since dst_clone() is done by other cpus, involving cache line ping pongs. It seems right place to release dst is in dev_hard_start_xmit(), for most devices but ones that are virtual, and some exceptions. David Miller suggested to define a new device flag, set in alloc_netdev_mq() (so that most devices set it at init time), and carefuly unset in devices which dont want a NULL skb->dst in their ndo_start_xmit(). List of devices that must clear this flag is : - loopback device, because it calls netif_rx() and quoting Patrick : "ip_route_input() doesn't accept loopback addresses, so loopback packets already need to have a dst_entry attached." - appletalk/ipddp.c : needs skb->dst in its xmit function - And all devices that call again dev_queue_xmit() from their xmit function (as some classifiers need skb->dst) : bonding, vlan, macvlan, eql, ifb, hdlc_fr Signed-off-by: Eric Dumazet <dada1@cosmosbay.com> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
parent
496a60cdcd
commit
93f154b594
|
@ -71,6 +71,7 @@ static struct net_device * __init ipddp_init(void)
|
|||
if (!dev)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
strcpy(dev->name, "ipddp%d");
|
||||
|
||||
if (version_printed++ == 0)
|
||||
|
|
|
@ -5148,6 +5148,7 @@ int bond_create(char *name, struct bond_params *params)
|
|||
goto out_rtnl;
|
||||
}
|
||||
|
||||
bond_dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
if (!name) {
|
||||
res = dev_alloc_name(bond_dev, "bond%d");
|
||||
if (res < 0)
|
||||
|
|
|
@ -194,6 +194,7 @@ static void __init eql_setup(struct net_device *dev)
|
|||
|
||||
dev->type = ARPHRD_SLIP;
|
||||
dev->tx_queue_len = 5; /* Hands them off fast */
|
||||
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
}
|
||||
|
||||
static int eql_open(struct net_device *dev)
|
||||
|
|
|
@ -156,6 +156,7 @@ static void ifb_setup(struct net_device *dev)
|
|||
|
||||
dev->flags |= IFF_NOARP;
|
||||
dev->flags &= ~IFF_MULTICAST;
|
||||
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
random_ether_addr(dev->dev_addr);
|
||||
}
|
||||
|
||||
|
|
|
@ -170,6 +170,7 @@ static void loopback_setup(struct net_device *dev)
|
|||
dev->tx_queue_len = 0;
|
||||
dev->type = ARPHRD_LOOPBACK; /* 0x0001*/
|
||||
dev->flags = IFF_LOOPBACK;
|
||||
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
dev->features = NETIF_F_SG | NETIF_F_FRAGLIST
|
||||
| NETIF_F_TSO
|
||||
| NETIF_F_NO_CSUM
|
||||
|
|
|
@ -414,6 +414,7 @@ static void macvlan_setup(struct net_device *dev)
|
|||
{
|
||||
ether_setup(dev);
|
||||
|
||||
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
dev->netdev_ops = &macvlan_netdev_ops;
|
||||
dev->destructor = free_netdev;
|
||||
dev->header_ops = &macvlan_hard_header_ops,
|
||||
|
|
|
@ -1054,6 +1054,7 @@ static void pvc_setup(struct net_device *dev)
|
|||
dev->flags = IFF_POINTOPOINT;
|
||||
dev->hard_header_len = 10;
|
||||
dev->addr_len = 2;
|
||||
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
}
|
||||
|
||||
static const struct net_device_ops pvc_ops = {
|
||||
|
|
|
@ -67,6 +67,9 @@
|
|||
#define IFF_ISATAP 0x80 /* ISATAP interface (RFC4214) */
|
||||
#define IFF_MASTER_ARPMON 0x100 /* bonding master, ARP mon in use */
|
||||
#define IFF_WAN_HDLC 0x200 /* WAN HDLC device */
|
||||
#define IFF_XMIT_DST_RELEASE 0x400 /* dev_hard_start_xmit() is allowed to
|
||||
* release skb->dst
|
||||
*/
|
||||
|
||||
#define IF_GET_IFACE 0x0001 /* for querying only */
|
||||
#define IF_GET_PROTO 0x0002
|
||||
|
|
|
@ -739,6 +739,7 @@ void vlan_setup(struct net_device *dev)
|
|||
ether_setup(dev);
|
||||
|
||||
dev->priv_flags |= IFF_802_1Q_VLAN;
|
||||
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
|
||||
dev->tx_queue_len = 0;
|
||||
|
||||
dev->netdev_ops = &vlan_netdev_ops;
|
||||
|
|
|
@ -1688,6 +1688,14 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
|
|||
goto gso;
|
||||
}
|
||||
|
||||
/*
|
||||
* If device doesnt need skb->dst, release it right now while
|
||||
* its hot in this cpu cache
|
||||
*/
|
||||
if ((dev->priv_flags & IFF_XMIT_DST_RELEASE) && skb->dst) {
|
||||
dst_release(skb->dst);
|
||||
skb->dst = NULL;
|
||||
}
|
||||
rc = ops->ndo_start_xmit(skb, dev);
|
||||
/*
|
||||
* TODO: if skb_orphan() was called by
|
||||
|
@ -5045,6 +5053,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
|
|||
netdev_init_queues(dev);
|
||||
|
||||
INIT_LIST_HEAD(&dev->napi_list);
|
||||
dev->priv_flags = IFF_XMIT_DST_RELEASE;
|
||||
setup(dev);
|
||||
strcpy(dev->name, name);
|
||||
return dev;
|
||||
|
|
Loading…
Reference in a new issue