![Eric Dumazet](/assets/img/avatar_default.png)
When many cpus compete for sending frames on a given qdisc, the qdisc spinlock suffers from very high contention. The cpu owning __QDISC_STATE_RUNNING bit has same priority to acquire the lock, and cannot dequeue packets fast enough, since it must wait for this lock for each dequeued packet. One solution to this problem is to force all cpus spinning on a second lock before trying to get the main lock, when/if they see __QDISC_STATE_RUNNING already set. The owning cpu then compete with at most one other cpu for the main lock, allowing for higher dequeueing rate. Based on a previous patch from Alexander Duyck. I added the heuristic to avoid the atomic in fast path, and put the new lock far away from the cache line used by the dequeue worker. Also try to release the busylock lock as late as possible. Tests with following script gave a boost from ~50.000 pps to ~600.000 pps on a dual quad core machine (E5450 @3.00GHz), tg3 driver. (A single netperf flow can reach ~800.000 pps on this platform) for j in `seq 0 3`; do for i in `seq 0 7`; do netperf -H 192.168.0.1 -t UDP_STREAM -l 60 -N -T $i -- -m 6 & done done Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Acked-by: Alexander Duyck <alexander.h.duyck@intel.com> Signed-off-by: David S. Miller <davem@davemloft.net>
870 lines
20 KiB
C
870 lines
20 KiB
C
/*
|
|
* net/sched/sch_generic.c Generic packet scheduler routines.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*
|
|
* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
|
|
* Jamal Hadi Salim, <hadi@cyberus.ca> 990601
|
|
* - Ingress support
|
|
*/
|
|
|
|
#include <linux/bitops.h>
|
|
#include <linux/module.h>
|
|
#include <linux/types.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/string.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/netdevice.h>
|
|
#include <linux/skbuff.h>
|
|
#include <linux/rtnetlink.h>
|
|
#include <linux/init.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/list.h>
|
|
#include <linux/slab.h>
|
|
#include <net/pkt_sched.h>
|
|
#include <net/dst.h>
|
|
|
|
/* Main transmission queue. */
|
|
|
|
/* Modifications to data participating in scheduling must be protected with
|
|
* qdisc_lock(qdisc) spinlock.
|
|
*
|
|
* The idea is the following:
|
|
* - enqueue, dequeue are serialized via qdisc root lock
|
|
* - ingress filtering is also serialized via qdisc root lock
|
|
* - updates to tree and tree walking are only done under the rtnl mutex.
|
|
*/
|
|
|
|
static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
|
|
{
|
|
skb_dst_force(skb);
|
|
q->gso_skb = skb;
|
|
q->qstats.requeues++;
|
|
q->q.qlen++; /* it's still part of the queue */
|
|
__netif_schedule(q);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static inline struct sk_buff *dequeue_skb(struct Qdisc *q)
|
|
{
|
|
struct sk_buff *skb = q->gso_skb;
|
|
|
|
if (unlikely(skb)) {
|
|
struct net_device *dev = qdisc_dev(q);
|
|
struct netdev_queue *txq;
|
|
|
|
/* check the reason of requeuing without tx lock first */
|
|
txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
|
|
if (!netif_tx_queue_stopped(txq) &&
|
|
!netif_tx_queue_frozen(txq)) {
|
|
q->gso_skb = NULL;
|
|
q->q.qlen--;
|
|
} else
|
|
skb = NULL;
|
|
} else {
|
|
skb = q->dequeue(q);
|
|
}
|
|
|
|
return skb;
|
|
}
|
|
|
|
static inline int handle_dev_cpu_collision(struct sk_buff *skb,
|
|
struct netdev_queue *dev_queue,
|
|
struct Qdisc *q)
|
|
{
|
|
int ret;
|
|
|
|
if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) {
|
|
/*
|
|
* Same CPU holding the lock. It may be a transient
|
|
* configuration error, when hard_start_xmit() recurses. We
|
|
* detect it by checking xmit owner and drop the packet when
|
|
* deadloop is detected. Return OK to try the next skb.
|
|
*/
|
|
kfree_skb(skb);
|
|
if (net_ratelimit())
|
|
printk(KERN_WARNING "Dead loop on netdevice %s, "
|
|
"fix it urgently!\n", dev_queue->dev->name);
|
|
ret = qdisc_qlen(q);
|
|
} else {
|
|
/*
|
|
* Another cpu is holding lock, requeue & delay xmits for
|
|
* some time.
|
|
*/
|
|
__get_cpu_var(softnet_data).cpu_collision++;
|
|
ret = dev_requeue_skb(skb, q);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Transmit one skb, and handle the return status as required. Holding the
|
|
* __QDISC_STATE_RUNNING bit guarantees that only one CPU can execute this
|
|
* function.
|
|
*
|
|
* Returns to the caller:
|
|
* 0 - queue is empty or throttled.
|
|
* >0 - queue is not empty.
|
|
*/
|
|
int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
|
|
struct net_device *dev, struct netdev_queue *txq,
|
|
spinlock_t *root_lock)
|
|
{
|
|
int ret = NETDEV_TX_BUSY;
|
|
|
|
/* And release qdisc */
|
|
spin_unlock(root_lock);
|
|
|
|
HARD_TX_LOCK(dev, txq, smp_processor_id());
|
|
if (!netif_tx_queue_stopped(txq) && !netif_tx_queue_frozen(txq))
|
|
ret = dev_hard_start_xmit(skb, dev, txq);
|
|
|
|
HARD_TX_UNLOCK(dev, txq);
|
|
|
|
spin_lock(root_lock);
|
|
|
|
if (dev_xmit_complete(ret)) {
|
|
/* Driver sent out skb successfully or skb was consumed */
|
|
ret = qdisc_qlen(q);
|
|
} else if (ret == NETDEV_TX_LOCKED) {
|
|
/* Driver try lock failed */
|
|
ret = handle_dev_cpu_collision(skb, txq, q);
|
|
} else {
|
|
/* Driver returned NETDEV_TX_BUSY - requeue skb */
|
|
if (unlikely (ret != NETDEV_TX_BUSY && net_ratelimit()))
|
|
printk(KERN_WARNING "BUG %s code %d qlen %d\n",
|
|
dev->name, ret, q->q.qlen);
|
|
|
|
ret = dev_requeue_skb(skb, q);
|
|
}
|
|
|
|
if (ret && (netif_tx_queue_stopped(txq) ||
|
|
netif_tx_queue_frozen(txq)))
|
|
ret = 0;
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* NOTE: Called under qdisc_lock(q) with locally disabled BH.
|
|
*
|
|
* __QDISC_STATE_RUNNING guarantees only one CPU can process
|
|
* this qdisc at a time. qdisc_lock(q) serializes queue accesses for
|
|
* this queue.
|
|
*
|
|
* netif_tx_lock serializes accesses to device driver.
|
|
*
|
|
* qdisc_lock(q) and netif_tx_lock are mutually exclusive,
|
|
* if one is grabbed, another must be free.
|
|
*
|
|
* Note, that this procedure can be called by a watchdog timer
|
|
*
|
|
* Returns to the caller:
|
|
* 0 - queue is empty or throttled.
|
|
* >0 - queue is not empty.
|
|
*
|
|
*/
|
|
static inline int qdisc_restart(struct Qdisc *q)
|
|
{
|
|
struct netdev_queue *txq;
|
|
struct net_device *dev;
|
|
spinlock_t *root_lock;
|
|
struct sk_buff *skb;
|
|
|
|
/* Dequeue packet */
|
|
skb = dequeue_skb(q);
|
|
if (unlikely(!skb))
|
|
return 0;
|
|
WARN_ON_ONCE(skb_dst_is_noref(skb));
|
|
root_lock = qdisc_lock(q);
|
|
dev = qdisc_dev(q);
|
|
txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
|
|
|
|
return sch_direct_xmit(skb, q, dev, txq, root_lock);
|
|
}
|
|
|
|
void __qdisc_run(struct Qdisc *q)
|
|
{
|
|
unsigned long start_time = jiffies;
|
|
|
|
while (qdisc_restart(q)) {
|
|
/*
|
|
* Postpone processing if
|
|
* 1. another process needs the CPU;
|
|
* 2. we've been doing it for too long.
|
|
*/
|
|
if (need_resched() || jiffies != start_time) {
|
|
__netif_schedule(q);
|
|
break;
|
|
}
|
|
}
|
|
|
|
qdisc_run_end(q);
|
|
}
|
|
|
|
unsigned long dev_trans_start(struct net_device *dev)
|
|
{
|
|
unsigned long val, res = dev->trans_start;
|
|
unsigned int i;
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
val = netdev_get_tx_queue(dev, i)->trans_start;
|
|
if (val && time_after(val, res))
|
|
res = val;
|
|
}
|
|
dev->trans_start = res;
|
|
return res;
|
|
}
|
|
EXPORT_SYMBOL(dev_trans_start);
|
|
|
|
static void dev_watchdog(unsigned long arg)
|
|
{
|
|
struct net_device *dev = (struct net_device *)arg;
|
|
|
|
netif_tx_lock(dev);
|
|
if (!qdisc_tx_is_noop(dev)) {
|
|
if (netif_device_present(dev) &&
|
|
netif_running(dev) &&
|
|
netif_carrier_ok(dev)) {
|
|
int some_queue_timedout = 0;
|
|
unsigned int i;
|
|
unsigned long trans_start;
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
struct netdev_queue *txq;
|
|
|
|
txq = netdev_get_tx_queue(dev, i);
|
|
/*
|
|
* old device drivers set dev->trans_start
|
|
*/
|
|
trans_start = txq->trans_start ? : dev->trans_start;
|
|
if (netif_tx_queue_stopped(txq) &&
|
|
time_after(jiffies, (trans_start +
|
|
dev->watchdog_timeo))) {
|
|
some_queue_timedout = 1;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (some_queue_timedout) {
|
|
char drivername[64];
|
|
WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
|
|
dev->name, netdev_drivername(dev, drivername, 64), i);
|
|
dev->netdev_ops->ndo_tx_timeout(dev);
|
|
}
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
round_jiffies(jiffies +
|
|
dev->watchdog_timeo)))
|
|
dev_hold(dev);
|
|
}
|
|
}
|
|
netif_tx_unlock(dev);
|
|
|
|
dev_put(dev);
|
|
}
|
|
|
|
void __netdev_watchdog_up(struct net_device *dev)
|
|
{
|
|
if (dev->netdev_ops->ndo_tx_timeout) {
|
|
if (dev->watchdog_timeo <= 0)
|
|
dev->watchdog_timeo = 5*HZ;
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
round_jiffies(jiffies + dev->watchdog_timeo)))
|
|
dev_hold(dev);
|
|
}
|
|
}
|
|
|
|
static void dev_watchdog_up(struct net_device *dev)
|
|
{
|
|
__netdev_watchdog_up(dev);
|
|
}
|
|
|
|
static void dev_watchdog_down(struct net_device *dev)
|
|
{
|
|
netif_tx_lock_bh(dev);
|
|
if (del_timer(&dev->watchdog_timer))
|
|
dev_put(dev);
|
|
netif_tx_unlock_bh(dev);
|
|
}
|
|
|
|
/**
|
|
* netif_carrier_on - set carrier
|
|
* @dev: network device
|
|
*
|
|
* Device has detected that carrier.
|
|
*/
|
|
void netif_carrier_on(struct net_device *dev)
|
|
{
|
|
if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
return;
|
|
linkwatch_fire_event(dev);
|
|
if (netif_running(dev))
|
|
__netdev_watchdog_up(dev);
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(netif_carrier_on);
|
|
|
|
/**
|
|
* netif_carrier_off - clear carrier
|
|
* @dev: network device
|
|
*
|
|
* Device has detected loss of carrier.
|
|
*/
|
|
void netif_carrier_off(struct net_device *dev)
|
|
{
|
|
if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
return;
|
|
linkwatch_fire_event(dev);
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(netif_carrier_off);
|
|
|
|
/**
|
|
* netif_notify_peers - notify network peers about existence of @dev
|
|
* @dev: network device
|
|
*
|
|
* Generate traffic such that interested network peers are aware of
|
|
* @dev, such as by generating a gratuitous ARP. This may be used when
|
|
* a device wants to inform the rest of the network about some sort of
|
|
* reconfiguration such as a failover event or virtual machine
|
|
* migration.
|
|
*/
|
|
void netif_notify_peers(struct net_device *dev)
|
|
{
|
|
rtnl_lock();
|
|
call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
|
|
rtnl_unlock();
|
|
}
|
|
EXPORT_SYMBOL(netif_notify_peers);
|
|
|
|
/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
|
|
under all circumstances. It is difficult to invent anything faster or
|
|
cheaper.
|
|
*/
|
|
|
|
static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc)
|
|
{
|
|
kfree_skb(skb);
|
|
return NET_XMIT_CN;
|
|
}
|
|
|
|
static struct sk_buff *noop_dequeue(struct Qdisc * qdisc)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
struct Qdisc_ops noop_qdisc_ops __read_mostly = {
|
|
.id = "noop",
|
|
.priv_size = 0,
|
|
.enqueue = noop_enqueue,
|
|
.dequeue = noop_dequeue,
|
|
.peek = noop_dequeue,
|
|
.owner = THIS_MODULE,
|
|
};
|
|
|
|
static struct netdev_queue noop_netdev_queue = {
|
|
.qdisc = &noop_qdisc,
|
|
.qdisc_sleeping = &noop_qdisc,
|
|
};
|
|
|
|
struct Qdisc noop_qdisc = {
|
|
.enqueue = noop_enqueue,
|
|
.dequeue = noop_dequeue,
|
|
.flags = TCQ_F_BUILTIN,
|
|
.ops = &noop_qdisc_ops,
|
|
.list = LIST_HEAD_INIT(noop_qdisc.list),
|
|
.q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
|
|
.dev_queue = &noop_netdev_queue,
|
|
};
|
|
EXPORT_SYMBOL(noop_qdisc);
|
|
|
|
static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
|
|
.id = "noqueue",
|
|
.priv_size = 0,
|
|
.enqueue = noop_enqueue,
|
|
.dequeue = noop_dequeue,
|
|
.peek = noop_dequeue,
|
|
.owner = THIS_MODULE,
|
|
};
|
|
|
|
static struct Qdisc noqueue_qdisc;
|
|
static struct netdev_queue noqueue_netdev_queue = {
|
|
.qdisc = &noqueue_qdisc,
|
|
.qdisc_sleeping = &noqueue_qdisc,
|
|
};
|
|
|
|
static struct Qdisc noqueue_qdisc = {
|
|
.enqueue = NULL,
|
|
.dequeue = noop_dequeue,
|
|
.flags = TCQ_F_BUILTIN,
|
|
.ops = &noqueue_qdisc_ops,
|
|
.list = LIST_HEAD_INIT(noqueue_qdisc.list),
|
|
.q.lock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.q.lock),
|
|
.dev_queue = &noqueue_netdev_queue,
|
|
};
|
|
|
|
|
|
static const u8 prio2band[TC_PRIO_MAX+1] =
|
|
{ 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 };
|
|
|
|
/* 3-band FIFO queue: old style, but should be a bit faster than
|
|
generic prio+fifo combination.
|
|
*/
|
|
|
|
#define PFIFO_FAST_BANDS 3
|
|
|
|
/*
|
|
* Private data for a pfifo_fast scheduler containing:
|
|
* - queues for the three band
|
|
* - bitmap indicating which of the bands contain skbs
|
|
*/
|
|
struct pfifo_fast_priv {
|
|
u32 bitmap;
|
|
struct sk_buff_head q[PFIFO_FAST_BANDS];
|
|
};
|
|
|
|
/*
|
|
* Convert a bitmap to the first band number where an skb is queued, where:
|
|
* bitmap=0 means there are no skbs on any band.
|
|
* bitmap=1 means there is an skb on band 0.
|
|
* bitmap=7 means there are skbs on all 3 bands, etc.
|
|
*/
|
|
static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0};
|
|
|
|
static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv,
|
|
int band)
|
|
{
|
|
return priv->q + band;
|
|
}
|
|
|
|
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc* qdisc)
|
|
{
|
|
if (skb_queue_len(&qdisc->q) < qdisc_dev(qdisc)->tx_queue_len) {
|
|
int band = prio2band[skb->priority & TC_PRIO_MAX];
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
struct sk_buff_head *list = band2list(priv, band);
|
|
|
|
priv->bitmap |= (1 << band);
|
|
qdisc->q.qlen++;
|
|
return __qdisc_enqueue_tail(skb, qdisc, list);
|
|
}
|
|
|
|
return qdisc_drop(skb, qdisc);
|
|
}
|
|
|
|
static struct sk_buff *pfifo_fast_dequeue(struct Qdisc* qdisc)
|
|
{
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
int band = bitmap2band[priv->bitmap];
|
|
|
|
if (likely(band >= 0)) {
|
|
struct sk_buff_head *list = band2list(priv, band);
|
|
struct sk_buff *skb = __qdisc_dequeue_head(qdisc, list);
|
|
|
|
qdisc->q.qlen--;
|
|
if (skb_queue_empty(list))
|
|
priv->bitmap &= ~(1 << band);
|
|
|
|
return skb;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static struct sk_buff *pfifo_fast_peek(struct Qdisc* qdisc)
|
|
{
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
int band = bitmap2band[priv->bitmap];
|
|
|
|
if (band >= 0) {
|
|
struct sk_buff_head *list = band2list(priv, band);
|
|
|
|
return skb_peek(list);
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static void pfifo_fast_reset(struct Qdisc* qdisc)
|
|
{
|
|
int prio;
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
|
|
__qdisc_reset_queue(qdisc, band2list(priv, prio));
|
|
|
|
priv->bitmap = 0;
|
|
qdisc->qstats.backlog = 0;
|
|
qdisc->q.qlen = 0;
|
|
}
|
|
|
|
static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
|
|
{
|
|
struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
|
|
|
|
memcpy(&opt.priomap, prio2band, TC_PRIO_MAX+1);
|
|
NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
|
|
return skb->len;
|
|
|
|
nla_put_failure:
|
|
return -1;
|
|
}
|
|
|
|
static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
|
|
{
|
|
int prio;
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
|
|
skb_queue_head_init(band2list(priv, prio));
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct Qdisc_ops pfifo_fast_ops __read_mostly = {
|
|
.id = "pfifo_fast",
|
|
.priv_size = sizeof(struct pfifo_fast_priv),
|
|
.enqueue = pfifo_fast_enqueue,
|
|
.dequeue = pfifo_fast_dequeue,
|
|
.peek = pfifo_fast_peek,
|
|
.init = pfifo_fast_init,
|
|
.reset = pfifo_fast_reset,
|
|
.dump = pfifo_fast_dump,
|
|
.owner = THIS_MODULE,
|
|
};
|
|
|
|
struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
|
|
struct Qdisc_ops *ops)
|
|
{
|
|
void *p;
|
|
struct Qdisc *sch;
|
|
unsigned int size;
|
|
int err = -ENOBUFS;
|
|
|
|
/* ensure that the Qdisc and the private data are 64-byte aligned */
|
|
size = QDISC_ALIGN(sizeof(*sch));
|
|
size += ops->priv_size + (QDISC_ALIGNTO - 1);
|
|
|
|
p = kzalloc(size, GFP_KERNEL);
|
|
if (!p)
|
|
goto errout;
|
|
sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
|
|
sch->padded = (char *) sch - (char *) p;
|
|
|
|
INIT_LIST_HEAD(&sch->list);
|
|
skb_queue_head_init(&sch->q);
|
|
spin_lock_init(&sch->busylock);
|
|
sch->ops = ops;
|
|
sch->enqueue = ops->enqueue;
|
|
sch->dequeue = ops->dequeue;
|
|
sch->dev_queue = dev_queue;
|
|
dev_hold(qdisc_dev(sch));
|
|
atomic_set(&sch->refcnt, 1);
|
|
|
|
return sch;
|
|
errout:
|
|
return ERR_PTR(err);
|
|
}
|
|
|
|
struct Qdisc * qdisc_create_dflt(struct net_device *dev,
|
|
struct netdev_queue *dev_queue,
|
|
struct Qdisc_ops *ops,
|
|
unsigned int parentid)
|
|
{
|
|
struct Qdisc *sch;
|
|
|
|
sch = qdisc_alloc(dev_queue, ops);
|
|
if (IS_ERR(sch))
|
|
goto errout;
|
|
sch->parent = parentid;
|
|
|
|
if (!ops->init || ops->init(sch, NULL) == 0)
|
|
return sch;
|
|
|
|
qdisc_destroy(sch);
|
|
errout:
|
|
return NULL;
|
|
}
|
|
EXPORT_SYMBOL(qdisc_create_dflt);
|
|
|
|
/* Under qdisc_lock(qdisc) and BH! */
|
|
|
|
void qdisc_reset(struct Qdisc *qdisc)
|
|
{
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
|
|
|
if (ops->reset)
|
|
ops->reset(qdisc);
|
|
|
|
if (qdisc->gso_skb) {
|
|
kfree_skb(qdisc->gso_skb);
|
|
qdisc->gso_skb = NULL;
|
|
qdisc->q.qlen = 0;
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(qdisc_reset);
|
|
|
|
static void qdisc_rcu_free(struct rcu_head *head)
|
|
{
|
|
struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head);
|
|
|
|
kfree((char *) qdisc - qdisc->padded);
|
|
}
|
|
|
|
void qdisc_destroy(struct Qdisc *qdisc)
|
|
{
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN ||
|
|
!atomic_dec_and_test(&qdisc->refcnt))
|
|
return;
|
|
|
|
#ifdef CONFIG_NET_SCHED
|
|
qdisc_list_del(qdisc);
|
|
|
|
qdisc_put_stab(qdisc->stab);
|
|
#endif
|
|
gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est);
|
|
if (ops->reset)
|
|
ops->reset(qdisc);
|
|
if (ops->destroy)
|
|
ops->destroy(qdisc);
|
|
|
|
module_put(ops->owner);
|
|
dev_put(qdisc_dev(qdisc));
|
|
|
|
kfree_skb(qdisc->gso_skb);
|
|
/*
|
|
* gen_estimator est_timer() might access qdisc->q.lock,
|
|
* wait a RCU grace period before freeing qdisc.
|
|
*/
|
|
call_rcu(&qdisc->rcu_head, qdisc_rcu_free);
|
|
}
|
|
EXPORT_SYMBOL(qdisc_destroy);
|
|
|
|
/* Attach toplevel qdisc to device queue. */
|
|
struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
|
|
struct Qdisc *qdisc)
|
|
{
|
|
struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
|
|
spinlock_t *root_lock;
|
|
|
|
root_lock = qdisc_lock(oqdisc);
|
|
spin_lock_bh(root_lock);
|
|
|
|
/* Prune old scheduler */
|
|
if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
|
|
qdisc_reset(oqdisc);
|
|
|
|
/* ... and graft new one */
|
|
if (qdisc == NULL)
|
|
qdisc = &noop_qdisc;
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
|
|
|
|
spin_unlock_bh(root_lock);
|
|
|
|
return oqdisc;
|
|
}
|
|
|
|
static void attach_one_default_qdisc(struct net_device *dev,
|
|
struct netdev_queue *dev_queue,
|
|
void *_unused)
|
|
{
|
|
struct Qdisc *qdisc;
|
|
|
|
if (dev->tx_queue_len) {
|
|
qdisc = qdisc_create_dflt(dev, dev_queue,
|
|
&pfifo_fast_ops, TC_H_ROOT);
|
|
if (!qdisc) {
|
|
printk(KERN_INFO "%s: activation failed\n", dev->name);
|
|
return;
|
|
}
|
|
|
|
/* Can by-pass the queue discipline for default qdisc */
|
|
qdisc->flags |= TCQ_F_CAN_BYPASS;
|
|
} else {
|
|
qdisc = &noqueue_qdisc;
|
|
}
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
}
|
|
|
|
static void attach_default_qdiscs(struct net_device *dev)
|
|
{
|
|
struct netdev_queue *txq;
|
|
struct Qdisc *qdisc;
|
|
|
|
txq = netdev_get_tx_queue(dev, 0);
|
|
|
|
if (!netif_is_multiqueue(dev) || dev->tx_queue_len == 0) {
|
|
netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
|
|
dev->qdisc = txq->qdisc_sleeping;
|
|
atomic_inc(&dev->qdisc->refcnt);
|
|
} else {
|
|
qdisc = qdisc_create_dflt(dev, txq, &mq_qdisc_ops, TC_H_ROOT);
|
|
if (qdisc) {
|
|
qdisc->ops->attach(qdisc);
|
|
dev->qdisc = qdisc;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void transition_one_qdisc(struct net_device *dev,
|
|
struct netdev_queue *dev_queue,
|
|
void *_need_watchdog)
|
|
{
|
|
struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
|
|
int *need_watchdog_p = _need_watchdog;
|
|
|
|
if (!(new_qdisc->flags & TCQ_F_BUILTIN))
|
|
clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
|
|
|
|
rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
|
|
if (need_watchdog_p && new_qdisc != &noqueue_qdisc) {
|
|
dev_queue->trans_start = 0;
|
|
*need_watchdog_p = 1;
|
|
}
|
|
}
|
|
|
|
void dev_activate(struct net_device *dev)
|
|
{
|
|
int need_watchdog;
|
|
|
|
/* No queueing discipline is attached to device;
|
|
create default one i.e. pfifo_fast for devices,
|
|
which need queueing and noqueue_qdisc for
|
|
virtual interfaces
|
|
*/
|
|
|
|
if (dev->qdisc == &noop_qdisc)
|
|
attach_default_qdiscs(dev);
|
|
|
|
if (!netif_carrier_ok(dev))
|
|
/* Delay activation until next carrier-on event */
|
|
return;
|
|
|
|
need_watchdog = 0;
|
|
netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
|
|
transition_one_qdisc(dev, &dev->rx_queue, NULL);
|
|
|
|
if (need_watchdog) {
|
|
dev->trans_start = jiffies;
|
|
dev_watchdog_up(dev);
|
|
}
|
|
}
|
|
|
|
static void dev_deactivate_queue(struct net_device *dev,
|
|
struct netdev_queue *dev_queue,
|
|
void *_qdisc_default)
|
|
{
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
|
struct Qdisc *qdisc;
|
|
|
|
qdisc = dev_queue->qdisc;
|
|
if (qdisc) {
|
|
spin_lock_bh(qdisc_lock(qdisc));
|
|
|
|
if (!(qdisc->flags & TCQ_F_BUILTIN))
|
|
set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
|
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
|
qdisc_reset(qdisc);
|
|
|
|
spin_unlock_bh(qdisc_lock(qdisc));
|
|
}
|
|
}
|
|
|
|
static bool some_qdisc_is_busy(struct net_device *dev)
|
|
{
|
|
unsigned int i;
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
struct netdev_queue *dev_queue;
|
|
spinlock_t *root_lock;
|
|
struct Qdisc *q;
|
|
int val;
|
|
|
|
dev_queue = netdev_get_tx_queue(dev, i);
|
|
q = dev_queue->qdisc_sleeping;
|
|
root_lock = qdisc_lock(q);
|
|
|
|
spin_lock_bh(root_lock);
|
|
|
|
val = (qdisc_is_running(q) ||
|
|
test_bit(__QDISC_STATE_SCHED, &q->state));
|
|
|
|
spin_unlock_bh(root_lock);
|
|
|
|
if (val)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void dev_deactivate(struct net_device *dev)
|
|
{
|
|
netdev_for_each_tx_queue(dev, dev_deactivate_queue, &noop_qdisc);
|
|
dev_deactivate_queue(dev, &dev->rx_queue, &noop_qdisc);
|
|
|
|
dev_watchdog_down(dev);
|
|
|
|
/* Wait for outstanding qdisc-less dev_queue_xmit calls. */
|
|
synchronize_rcu();
|
|
|
|
/* Wait for outstanding qdisc_run calls. */
|
|
while (some_qdisc_is_busy(dev))
|
|
yield();
|
|
}
|
|
|
|
static void dev_init_scheduler_queue(struct net_device *dev,
|
|
struct netdev_queue *dev_queue,
|
|
void *_qdisc)
|
|
{
|
|
struct Qdisc *qdisc = _qdisc;
|
|
|
|
dev_queue->qdisc = qdisc;
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
}
|
|
|
|
void dev_init_scheduler(struct net_device *dev)
|
|
{
|
|
dev->qdisc = &noop_qdisc;
|
|
netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
|
|
dev_init_scheduler_queue(dev, &dev->rx_queue, &noop_qdisc);
|
|
|
|
setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev);
|
|
}
|
|
|
|
static void shutdown_scheduler_queue(struct net_device *dev,
|
|
struct netdev_queue *dev_queue,
|
|
void *_qdisc_default)
|
|
{
|
|
struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
|
|
|
if (qdisc) {
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
|
dev_queue->qdisc_sleeping = qdisc_default;
|
|
|
|
qdisc_destroy(qdisc);
|
|
}
|
|
}
|
|
|
|
void dev_shutdown(struct net_device *dev)
|
|
{
|
|
netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
|
|
shutdown_scheduler_queue(dev, &dev->rx_queue, &noop_qdisc);
|
|
qdisc_destroy(dev->qdisc);
|
|
dev->qdisc = &noop_qdisc;
|
|
|
|
WARN_ON(timer_pending(&dev->watchdog_timer));
|
|
}
|