2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* net/sched/sch_generic.c Generic packet scheduler routines.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
|
|
|
|
* Jamal Hadi Salim, <hadi@cyberus.ca> 990601
|
|
|
|
* - Ingress support
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <linux/rtnetlink.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
#include <linux/list.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 02:04:11 -06:00
|
|
|
#include <linux/slab.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <net/pkt_sched.h>
|
2010-05-11 17:19:48 -06:00
|
|
|
#include <net/dst.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Main transmission queue. */
|
|
|
|
|
2007-04-16 18:02:10 -06:00
|
|
|
/* Modifications to data participating in scheduling must be protected with
|
2008-08-02 21:02:43 -06:00
|
|
|
* qdisc_lock(qdisc) spinlock.
|
2007-04-16 18:02:10 -06:00
|
|
|
*
|
|
|
|
* The idea is the following:
|
2008-07-16 04:22:39 -06:00
|
|
|
* - enqueue, dequeue are serialized via qdisc root lock
|
|
|
|
* - ingress filtering is also serialized via qdisc root lock
|
2007-04-16 18:02:10 -06:00
|
|
|
* - updates to tree and tree walking are only done under the rtnl mutex.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
2008-07-16 03:15:04 -06:00
|
|
|
static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
|
2007-06-10 18:31:24 -06:00
|
|
|
{
|
2010-05-11 17:19:48 -06:00
|
|
|
skb_dst_force(skb);
|
2008-10-06 11:41:50 -06:00
|
|
|
q->gso_skb = skb;
|
2008-10-08 12:36:22 -06:00
|
|
|
q->qstats.requeues++;
|
2009-08-05 19:44:21 -06:00
|
|
|
q->q.qlen++; /* it's still part of the queue */
|
2008-07-16 03:15:04 -06:00
|
|
|
__netif_schedule(q);
|
2008-10-06 11:41:50 -06:00
|
|
|
|
2007-06-10 18:31:24 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-07-15 21:14:35 -06:00
|
|
|
static inline struct sk_buff *dequeue_skb(struct Qdisc *q)
|
2007-06-10 18:31:24 -06:00
|
|
|
{
|
2008-10-06 10:54:39 -06:00
|
|
|
struct sk_buff *skb = q->gso_skb;
|
|
|
|
|
2008-09-22 23:16:23 -06:00
|
|
|
if (unlikely(skb)) {
|
|
|
|
struct net_device *dev = qdisc_dev(q);
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
|
|
|
|
/* check the reason of requeuing without tx lock first */
|
|
|
|
txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
|
2010-11-23 03:42:02 -07:00
|
|
|
if (!netif_tx_queue_frozen_or_stopped(txq)) {
|
2008-10-06 11:41:50 -06:00
|
|
|
q->gso_skb = NULL;
|
2009-08-05 19:44:21 -06:00
|
|
|
q->q.qlen--;
|
|
|
|
} else
|
2008-09-22 23:16:23 -06:00
|
|
|
skb = NULL;
|
|
|
|
} else {
|
2007-06-10 18:31:24 -06:00
|
|
|
skb = q->dequeue(q);
|
2008-09-22 23:16:23 -06:00
|
|
|
}
|
2007-06-10 18:31:24 -06:00
|
|
|
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
static inline int handle_dev_cpu_collision(struct sk_buff *skb,
|
2008-07-09 00:10:33 -06:00
|
|
|
struct netdev_queue *dev_queue,
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
struct Qdisc *q)
|
2007-06-10 18:31:24 -06:00
|
|
|
{
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
int ret;
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2008-07-09 00:13:53 -06:00
|
|
|
if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) {
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
/*
|
|
|
|
* Same CPU holding the lock. It may be a transient
|
|
|
|
* configuration error, when hard_start_xmit() recurses. We
|
|
|
|
* detect it by checking xmit owner and drop the packet when
|
|
|
|
* deadloop is detected. Return OK to try the next skb.
|
|
|
|
*/
|
2007-06-10 18:31:24 -06:00
|
|
|
kfree_skb(skb);
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
if (net_ratelimit())
|
|
|
|
printk(KERN_WARNING "Dead loop on netdevice %s, "
|
2008-07-09 00:13:53 -06:00
|
|
|
"fix it urgently!\n", dev_queue->dev->name);
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
ret = qdisc_qlen(q);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Another cpu is holding lock, requeue & delay xmits for
|
|
|
|
* some time.
|
|
|
|
*/
|
2010-07-19 04:48:49 -06:00
|
|
|
__this_cpu_inc(softnet_data.cpu_collision);
|
2008-07-16 03:15:04 -06:00
|
|
|
ret = dev_requeue_skb(skb, q);
|
2007-06-10 18:31:24 -06:00
|
|
|
}
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
return ret;
|
2007-06-10 18:31:24 -06:00
|
|
|
}
|
|
|
|
|
2007-02-09 07:25:16 -07:00
|
|
|
/*
|
2009-08-05 19:44:21 -06:00
|
|
|
* Transmit one skb, and handle the return status as required. Holding the
|
|
|
|
* __QDISC_STATE_RUNNING bit guarantees that only one CPU can execute this
|
|
|
|
* function.
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
*
|
|
|
|
* Returns to the caller:
|
|
|
|
* 0 - queue is empty or throttled.
|
|
|
|
* >0 - queue is not empty.
|
|
|
|
*/
|
2009-08-05 19:44:21 -06:00
|
|
|
int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
|
|
|
|
struct net_device *dev, struct netdev_queue *txq,
|
|
|
|
spinlock_t *root_lock)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-11-13 21:40:55 -07:00
|
|
|
int ret = NETDEV_TX_BUSY;
|
2008-07-16 02:42:40 -06:00
|
|
|
|
|
|
|
/* And release qdisc */
|
|
|
|
spin_unlock(root_lock);
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2008-07-09 00:13:53 -06:00
|
|
|
HARD_TX_LOCK(dev, txq, smp_processor_id());
|
2010-11-23 03:42:02 -07:00
|
|
|
if (!netif_tx_queue_frozen_or_stopped(txq))
|
2008-07-17 02:56:23 -06:00
|
|
|
ret = dev_hard_start_xmit(skb, dev, txq);
|
2009-11-09 23:14:14 -07:00
|
|
|
|
2008-07-09 00:13:53 -06:00
|
|
|
HARD_TX_UNLOCK(dev, txq);
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2008-07-16 02:42:40 -06:00
|
|
|
spin_lock(root_lock);
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2009-11-15 00:20:12 -07:00
|
|
|
if (dev_xmit_complete(ret)) {
|
|
|
|
/* Driver sent out skb successfully or skb was consumed */
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
ret = qdisc_qlen(q);
|
2009-11-15 00:20:12 -07:00
|
|
|
} else if (ret == NETDEV_TX_LOCKED) {
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
/* Driver try lock failed */
|
2008-07-09 00:12:38 -06:00
|
|
|
ret = handle_dev_cpu_collision(skb, txq, q);
|
2009-11-15 00:20:12 -07:00
|
|
|
} else {
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
/* Driver returned NETDEV_TX_BUSY - requeue skb */
|
|
|
|
if (unlikely (ret != NETDEV_TX_BUSY && net_ratelimit()))
|
|
|
|
printk(KERN_WARNING "BUG %s code %d qlen %d\n",
|
|
|
|
dev->name, ret, q->q.qlen);
|
|
|
|
|
2008-07-16 03:15:04 -06:00
|
|
|
ret = dev_requeue_skb(skb, q);
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
}
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2010-11-23 03:42:02 -07:00
|
|
|
if (ret && netif_tx_queue_frozen_or_stopped(txq))
|
2008-07-16 03:15:04 -06:00
|
|
|
ret = 0;
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
return ret;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2009-08-05 19:44:21 -06:00
|
|
|
/*
|
|
|
|
* NOTE: Called under qdisc_lock(q) with locally disabled BH.
|
|
|
|
*
|
|
|
|
* __QDISC_STATE_RUNNING guarantees only one CPU can process
|
|
|
|
* this qdisc at a time. qdisc_lock(q) serializes queue accesses for
|
|
|
|
* this queue.
|
|
|
|
*
|
|
|
|
* netif_tx_lock serializes accesses to device driver.
|
|
|
|
*
|
|
|
|
* qdisc_lock(q) and netif_tx_lock are mutually exclusive,
|
|
|
|
* if one is grabbed, another must be free.
|
|
|
|
*
|
|
|
|
* Note, that this procedure can be called by a watchdog timer
|
|
|
|
*
|
|
|
|
* Returns to the caller:
|
|
|
|
* 0 - queue is empty or throttled.
|
|
|
|
* >0 - queue is not empty.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int qdisc_restart(struct Qdisc *q)
|
|
|
|
{
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
struct net_device *dev;
|
|
|
|
spinlock_t *root_lock;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
/* Dequeue packet */
|
|
|
|
skb = dequeue_skb(q);
|
|
|
|
if (unlikely(!skb))
|
|
|
|
return 0;
|
2010-05-11 17:19:48 -06:00
|
|
|
WARN_ON_ONCE(skb_dst_is_noref(skb));
|
2009-08-05 19:44:21 -06:00
|
|
|
root_lock = qdisc_lock(q);
|
|
|
|
dev = qdisc_dev(q);
|
|
|
|
txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
|
|
|
|
|
|
|
|
return sch_direct_xmit(skb, q, dev, txq, root_lock);
|
|
|
|
}
|
|
|
|
|
2008-07-16 03:15:04 -06:00
|
|
|
void __qdisc_run(struct Qdisc *q)
|
2006-06-20 00:57:59 -06:00
|
|
|
{
|
2008-03-28 17:25:26 -06:00
|
|
|
unsigned long start_time = jiffies;
|
|
|
|
|
2008-07-16 03:15:04 -06:00
|
|
|
while (qdisc_restart(q)) {
|
2008-03-28 17:25:26 -06:00
|
|
|
/*
|
|
|
|
* Postpone processing if
|
|
|
|
* 1. another process needs the CPU;
|
|
|
|
* 2. we've been doing it for too long.
|
|
|
|
*/
|
|
|
|
if (need_resched() || jiffies != start_time) {
|
2008-07-16 03:15:04 -06:00
|
|
|
__netif_schedule(q);
|
2007-05-10 05:55:14 -06:00
|
|
|
break;
|
2008-03-28 17:25:26 -06:00
|
|
|
}
|
|
|
|
}
|
2006-06-20 00:57:59 -06:00
|
|
|
|
2010-06-02 04:23:51 -06:00
|
|
|
qdisc_run_end(q);
|
2006-06-20 00:57:59 -06:00
|
|
|
}
|
|
|
|
|
2009-05-17 21:55:16 -06:00
|
|
|
unsigned long dev_trans_start(struct net_device *dev)
|
|
|
|
{
|
|
|
|
unsigned long val, res = dev->trans_start;
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
val = netdev_get_tx_queue(dev, i)->trans_start;
|
|
|
|
if (val && time_after(val, res))
|
|
|
|
res = val;
|
|
|
|
}
|
|
|
|
dev->trans_start = res;
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dev_trans_start);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static void dev_watchdog(unsigned long arg)
|
|
|
|
{
|
|
|
|
struct net_device *dev = (struct net_device *)arg;
|
|
|
|
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_lock(dev);
|
2008-07-17 01:34:19 -06:00
|
|
|
if (!qdisc_tx_is_noop(dev)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (netif_device_present(dev) &&
|
|
|
|
netif_running(dev) &&
|
|
|
|
netif_carrier_ok(dev)) {
|
2009-05-17 21:55:16 -06:00
|
|
|
int some_queue_timedout = 0;
|
2008-07-17 01:34:19 -06:00
|
|
|
unsigned int i;
|
2009-05-17 21:55:16 -06:00
|
|
|
unsigned long trans_start;
|
2008-07-17 01:34:19 -06:00
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
|
|
|
|
txq = netdev_get_tx_queue(dev, i);
|
2009-05-17 21:55:16 -06:00
|
|
|
/*
|
|
|
|
* old device drivers set dev->trans_start
|
|
|
|
*/
|
|
|
|
trans_start = txq->trans_start ? : dev->trans_start;
|
|
|
|
if (netif_tx_queue_stopped(txq) &&
|
|
|
|
time_after(jiffies, (trans_start +
|
|
|
|
dev->watchdog_timeo))) {
|
|
|
|
some_queue_timedout = 1;
|
2008-07-17 01:34:19 -06:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2006-05-16 16:02:12 -06:00
|
|
|
|
2009-05-17 21:55:16 -06:00
|
|
|
if (some_queue_timedout) {
|
2008-07-21 14:31:48 -06:00
|
|
|
char drivername[64];
|
2009-05-17 21:55:16 -06:00
|
|
|
WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
|
|
|
|
dev->name, netdev_drivername(dev, drivername, 64), i);
|
2008-11-19 22:32:24 -07:00
|
|
|
dev->netdev_ops->ndo_tx_timeout(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-07-17 01:34:19 -06:00
|
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
|
|
round_jiffies(jiffies +
|
|
|
|
dev->watchdog_timeo)))
|
2005-04-16 16:20:36 -06:00
|
|
|
dev_hold(dev);
|
|
|
|
}
|
|
|
|
}
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_unlock(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
dev_put(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __netdev_watchdog_up(struct net_device *dev)
|
|
|
|
{
|
2008-11-19 22:32:24 -07:00
|
|
|
if (dev->netdev_ops->ndo_tx_timeout) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (dev->watchdog_timeo <= 0)
|
|
|
|
dev->watchdog_timeo = 5*HZ;
|
2007-05-31 22:28:44 -06:00
|
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
|
|
round_jiffies(jiffies + dev->watchdog_timeo)))
|
2005-04-16 16:20:36 -06:00
|
|
|
dev_hold(dev);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dev_watchdog_up(struct net_device *dev)
|
|
|
|
{
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dev_watchdog_down(struct net_device *dev)
|
|
|
|
{
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_lock_bh(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (del_timer(&dev->watchdog_timer))
|
2006-03-20 23:32:28 -07:00
|
|
|
dev_put(dev);
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_unlock_bh(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 17:41:36 -06:00
|
|
|
/**
|
|
|
|
* netif_carrier_on - set carrier
|
|
|
|
* @dev: network device
|
|
|
|
*
|
|
|
|
* Device has detected that carrier.
|
|
|
|
*/
|
2005-08-11 16:32:53 -06:00
|
|
|
void netif_carrier_on(struct net_device *dev)
|
|
|
|
{
|
2007-10-18 00:26:43 -06:00
|
|
|
if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
2008-11-19 16:33:54 -07:00
|
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
|
|
return;
|
2005-08-11 16:32:53 -06:00
|
|
|
linkwatch_fire_event(dev);
|
2007-10-18 00:26:43 -06:00
|
|
|
if (netif_running(dev))
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
}
|
2005-08-11 16:32:53 -06:00
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(netif_carrier_on);
|
2005-08-11 16:32:53 -06:00
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 17:41:36 -06:00
|
|
|
/**
|
|
|
|
* netif_carrier_off - clear carrier
|
|
|
|
* @dev: network device
|
|
|
|
*
|
|
|
|
* Device has detected loss of carrier.
|
|
|
|
*/
|
2005-08-11 16:32:53 -06:00
|
|
|
void netif_carrier_off(struct net_device *dev)
|
|
|
|
{
|
2008-11-19 16:33:54 -07:00
|
|
|
if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
|
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
|
|
return;
|
2005-08-11 16:32:53 -06:00
|
|
|
linkwatch_fire_event(dev);
|
2008-11-19 16:33:54 -07:00
|
|
|
}
|
2005-08-11 16:32:53 -06:00
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(netif_carrier_off);
|
2005-08-11 16:32:53 -06:00
|
|
|
|
2010-05-25 18:09:42 -06:00
|
|
|
/**
|
|
|
|
* netif_notify_peers - notify network peers about existence of @dev
|
|
|
|
* @dev: network device
|
|
|
|
*
|
|
|
|
* Generate traffic such that interested network peers are aware of
|
|
|
|
* @dev, such as by generating a gratuitous ARP. This may be used when
|
|
|
|
* a device wants to inform the rest of the network about some sort of
|
|
|
|
* reconfiguration such as a failover event or virtual machine
|
|
|
|
* migration.
|
|
|
|
*/
|
|
|
|
void netif_notify_peers(struct net_device *dev)
|
|
|
|
{
|
|
|
|
rtnl_lock();
|
|
|
|
call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
|
|
|
|
rtnl_unlock();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(netif_notify_peers);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
|
|
|
|
under all circumstances. It is difficult to invent anything faster or
|
|
|
|
cheaper.
|
|
|
|
*/
|
|
|
|
|
2005-06-18 23:59:08 -06:00
|
|
|
static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
kfree_skb(skb);
|
|
|
|
return NET_XMIT_CN;
|
|
|
|
}
|
|
|
|
|
2005-06-18 23:59:08 -06:00
|
|
|
static struct sk_buff *noop_dequeue(struct Qdisc * qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2007-11-14 02:44:41 -07:00
|
|
|
struct Qdisc_ops noop_qdisc_ops __read_mostly = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.id = "noop",
|
|
|
|
.priv_size = 0,
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
2008-10-31 01:45:27 -06:00
|
|
|
.peek = noop_dequeue,
|
2005-04-16 16:20:36 -06:00
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
2008-07-16 02:42:40 -06:00
|
|
|
static struct netdev_queue noop_netdev_queue = {
|
|
|
|
.qdisc = &noop_qdisc,
|
2008-10-20 00:37:47 -06:00
|
|
|
.qdisc_sleeping = &noop_qdisc,
|
2008-07-16 02:42:40 -06:00
|
|
|
};
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
struct Qdisc noop_qdisc = {
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
.flags = TCQ_F_BUILTIN,
|
2007-02-09 07:25:16 -07:00
|
|
|
.ops = &noop_qdisc_ops,
|
2005-04-16 16:20:36 -06:00
|
|
|
.list = LIST_HEAD_INIT(noop_qdisc.list),
|
2008-07-17 01:53:03 -06:00
|
|
|
.q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
|
2008-07-16 02:42:40 -06:00
|
|
|
.dev_queue = &noop_netdev_queue,
|
2010-10-15 13:22:34 -06:00
|
|
|
.busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(noop_qdisc);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-11-14 02:44:41 -07:00
|
|
|
static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.id = "noqueue",
|
|
|
|
.priv_size = 0,
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
2008-10-31 01:45:27 -06:00
|
|
|
.peek = noop_dequeue,
|
2005-04-16 16:20:36 -06:00
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
2008-07-19 00:00:11 -06:00
|
|
|
static struct Qdisc noqueue_qdisc;
|
|
|
|
static struct netdev_queue noqueue_netdev_queue = {
|
|
|
|
.qdisc = &noqueue_qdisc,
|
2008-10-20 00:37:47 -06:00
|
|
|
.qdisc_sleeping = &noqueue_qdisc,
|
2008-07-19 00:00:11 -06:00
|
|
|
};
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static struct Qdisc noqueue_qdisc = {
|
|
|
|
.enqueue = NULL,
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
.flags = TCQ_F_BUILTIN,
|
|
|
|
.ops = &noqueue_qdisc_ops,
|
|
|
|
.list = LIST_HEAD_INIT(noqueue_qdisc.list),
|
2008-07-19 00:00:11 -06:00
|
|
|
.q.lock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.q.lock),
|
|
|
|
.dev_queue = &noqueue_netdev_queue,
|
2010-10-15 13:22:34 -06:00
|
|
|
.busylock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.busylock),
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2008-07-21 10:56:13 -06:00
|
|
|
static const u8 prio2band[TC_PRIO_MAX+1] =
|
|
|
|
{ 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 };
|
|
|
|
|
|
|
|
/* 3-band FIFO queue: old style, but should be a bit faster than
|
|
|
|
generic prio+fifo combination.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define PFIFO_FAST_BANDS 3
|
|
|
|
|
2009-08-18 15:55:59 -06:00
|
|
|
/*
|
|
|
|
* Private data for a pfifo_fast scheduler containing:
|
|
|
|
* - queues for the three band
|
|
|
|
* - bitmap indicating which of the bands contain skbs
|
|
|
|
*/
|
|
|
|
struct pfifo_fast_priv {
|
|
|
|
u32 bitmap;
|
|
|
|
struct sk_buff_head q[PFIFO_FAST_BANDS];
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert a bitmap to the first band number where an skb is queued, where:
|
|
|
|
* bitmap=0 means there are no skbs on any band.
|
|
|
|
* bitmap=1 means there is an skb on band 0.
|
|
|
|
* bitmap=7 means there are skbs on all 3 bands, etc.
|
|
|
|
*/
|
|
|
|
static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0};
|
|
|
|
|
|
|
|
static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv,
|
|
|
|
int band)
|
2008-07-21 10:56:13 -06:00
|
|
|
{
|
2009-08-18 15:55:59 -06:00
|
|
|
return priv->q + band;
|
2008-07-21 10:56:13 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc* qdisc)
|
2005-06-18 23:58:35 -06:00
|
|
|
{
|
2009-08-30 23:20:28 -06:00
|
|
|
if (skb_queue_len(&qdisc->q) < qdisc_dev(qdisc)->tx_queue_len) {
|
|
|
|
int band = prio2band[skb->priority & TC_PRIO_MAX];
|
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
|
|
struct sk_buff_head *list = band2list(priv, band);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-08-18 15:55:59 -06:00
|
|
|
priv->bitmap |= (1 << band);
|
2008-07-21 10:56:13 -06:00
|
|
|
qdisc->q.qlen++;
|
2005-06-18 23:58:15 -06:00
|
|
|
return __qdisc_enqueue_tail(skb, qdisc, list);
|
2008-07-21 10:56:13 -06:00
|
|
|
}
|
2005-06-18 23:58:15 -06:00
|
|
|
|
|
|
|
return qdisc_drop(skb, qdisc);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-21 10:56:13 -06:00
|
|
|
static struct sk_buff *pfifo_fast_dequeue(struct Qdisc* qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
|
|
int band = bitmap2band[priv->bitmap];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-08-18 15:55:59 -06:00
|
|
|
if (likely(band >= 0)) {
|
|
|
|
struct sk_buff_head *list = band2list(priv, band);
|
|
|
|
struct sk_buff *skb = __qdisc_dequeue_head(qdisc, list);
|
|
|
|
|
|
|
|
qdisc->q.qlen--;
|
|
|
|
if (skb_queue_empty(list))
|
|
|
|
priv->bitmap &= ~(1 << band);
|
|
|
|
|
|
|
|
return skb;
|
2008-07-21 10:56:13 -06:00
|
|
|
}
|
2005-06-18 23:58:53 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2008-10-31 01:45:27 -06:00
|
|
|
static struct sk_buff *pfifo_fast_peek(struct Qdisc* qdisc)
|
|
|
|
{
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
|
|
int band = bitmap2band[priv->bitmap];
|
|
|
|
|
|
|
|
if (band >= 0) {
|
|
|
|
struct sk_buff_head *list = band2list(priv, band);
|
2008-10-31 01:45:27 -06:00
|
|
|
|
2009-08-18 15:55:59 -06:00
|
|
|
return skb_peek(list);
|
2008-10-31 01:45:27 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2008-07-21 10:56:13 -06:00
|
|
|
static void pfifo_fast_reset(struct Qdisc* qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-07-21 10:56:13 -06:00
|
|
|
int prio;
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2008-07-21 10:56:13 -06:00
|
|
|
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
|
2009-08-18 15:55:59 -06:00
|
|
|
__qdisc_reset_queue(qdisc, band2list(priv, prio));
|
2008-07-21 10:56:13 -06:00
|
|
|
|
2009-08-18 15:55:59 -06:00
|
|
|
priv->bitmap = 0;
|
2005-06-18 23:58:15 -06:00
|
|
|
qdisc->qstats.backlog = 0;
|
2008-07-21 10:56:13 -06:00
|
|
|
qdisc->q.qlen = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-21 10:56:13 -06:00
|
|
|
static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
|
|
|
|
|
|
|
|
memcpy(&opt.priomap, prio2band, TC_PRIO_MAX+1);
|
|
|
|
NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
|
|
|
|
return skb->len;
|
|
|
|
|
|
|
|
nla_put_failure:
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
|
|
|
|
{
|
|
|
|
int prio;
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2008-07-21 10:56:13 -06:00
|
|
|
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
|
2009-08-18 15:55:59 -06:00
|
|
|
skb_queue_head_init(band2list(priv, prio));
|
2008-07-21 10:56:13 -06:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-09-06 02:58:51 -06:00
|
|
|
struct Qdisc_ops pfifo_fast_ops __read_mostly = {
|
2008-07-21 10:56:13 -06:00
|
|
|
.id = "pfifo_fast",
|
2009-08-18 15:55:59 -06:00
|
|
|
.priv_size = sizeof(struct pfifo_fast_priv),
|
2008-07-21 10:56:13 -06:00
|
|
|
.enqueue = pfifo_fast_enqueue,
|
|
|
|
.dequeue = pfifo_fast_dequeue,
|
2008-10-31 01:45:27 -06:00
|
|
|
.peek = pfifo_fast_peek,
|
2008-07-21 10:56:13 -06:00
|
|
|
.init = pfifo_fast_init,
|
|
|
|
.reset = pfifo_fast_reset,
|
|
|
|
.dump = pfifo_fast_dump,
|
2005-04-16 16:20:36 -06:00
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
2008-07-08 18:06:30 -06:00
|
|
|
struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
|
2008-07-08 17:55:56 -06:00
|
|
|
struct Qdisc_ops *ops)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
struct Qdisc *sch;
|
2005-07-05 15:15:09 -06:00
|
|
|
unsigned int size;
|
|
|
|
int err = -ENOBUFS;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-03-31 01:06:04 -06:00
|
|
|
/* ensure that the Qdisc and the private data are 64-byte aligned */
|
2005-07-05 15:15:09 -06:00
|
|
|
size = QDISC_ALIGN(sizeof(*sch));
|
|
|
|
size += ops->priv_size + (QDISC_ALIGNTO - 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-11-29 01:14:37 -07:00
|
|
|
p = kzalloc_node(size, GFP_KERNEL,
|
|
|
|
netdev_queue_numa_node_read(dev_queue));
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!p)
|
2005-07-05 15:15:09 -06:00
|
|
|
goto errout;
|
|
|
|
sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
|
|
|
|
sch->padded = (char *) sch - (char *) p;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
INIT_LIST_HEAD(&sch->list);
|
|
|
|
skb_queue_head_init(&sch->q);
|
2010-06-02 06:09:29 -06:00
|
|
|
spin_lock_init(&sch->busylock);
|
2005-04-16 16:20:36 -06:00
|
|
|
sch->ops = ops;
|
|
|
|
sch->enqueue = ops->enqueue;
|
|
|
|
sch->dequeue = ops->dequeue;
|
2008-07-08 17:55:56 -06:00
|
|
|
sch->dev_queue = dev_queue;
|
2008-07-08 18:06:30 -06:00
|
|
|
dev_hold(qdisc_dev(sch));
|
2005-04-16 16:20:36 -06:00
|
|
|
atomic_set(&sch->refcnt, 1);
|
2005-07-05 15:15:09 -06:00
|
|
|
|
|
|
|
return sch;
|
|
|
|
errout:
|
2008-06-27 20:51:35 -06:00
|
|
|
return ERR_PTR(err);
|
2005-07-05 15:15:09 -06:00
|
|
|
}
|
|
|
|
|
2010-10-16 07:04:08 -06:00
|
|
|
struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
|
|
|
|
struct Qdisc_ops *ops, unsigned int parentid)
|
2005-07-05 15:15:09 -06:00
|
|
|
{
|
|
|
|
struct Qdisc *sch;
|
2007-02-09 07:25:16 -07:00
|
|
|
|
2008-07-08 18:06:30 -06:00
|
|
|
sch = qdisc_alloc(dev_queue, ops);
|
2005-07-05 15:15:09 -06:00
|
|
|
if (IS_ERR(sch))
|
|
|
|
goto errout;
|
2006-11-29 18:35:18 -07:00
|
|
|
sch->parent = parentid;
|
2005-07-05 15:15:09 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!ops->init || ops->init(sch, NULL) == 0)
|
|
|
|
return sch;
|
|
|
|
|
2005-08-23 11:12:44 -06:00
|
|
|
qdisc_destroy(sch);
|
2005-07-05 15:15:09 -06:00
|
|
|
errout:
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(qdisc_create_dflt);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-08-02 21:02:43 -06:00
|
|
|
/* Under qdisc_lock(qdisc) and BH! */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
void qdisc_reset(struct Qdisc *qdisc)
|
|
|
|
{
|
2007-11-14 02:44:41 -07:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (ops->reset)
|
|
|
|
ops->reset(qdisc);
|
2008-11-03 03:52:50 -07:00
|
|
|
|
2009-08-05 19:44:21 -06:00
|
|
|
if (qdisc->gso_skb) {
|
|
|
|
kfree_skb(qdisc->gso_skb);
|
|
|
|
qdisc->gso_skb = NULL;
|
|
|
|
qdisc->q.qlen = 0;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(qdisc_reset);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-03-31 01:06:04 -06:00
|
|
|
static void qdisc_rcu_free(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head);
|
|
|
|
|
|
|
|
kfree((char *) qdisc - qdisc->padded);
|
|
|
|
}
|
|
|
|
|
2008-08-17 23:31:26 -06:00
|
|
|
void qdisc_destroy(struct Qdisc *qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-07-17 01:47:45 -06:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
|
|
|
|
2008-08-17 23:31:26 -06:00
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN ||
|
|
|
|
!atomic_dec_and_test(&qdisc->refcnt))
|
|
|
|
return;
|
|
|
|
|
2008-07-20 19:13:01 -06:00
|
|
|
#ifdef CONFIG_NET_SCHED
|
2008-08-22 04:24:05 -06:00
|
|
|
qdisc_list_del(qdisc);
|
|
|
|
|
2008-07-20 01:08:47 -06:00
|
|
|
qdisc_put_stab(qdisc->stab);
|
2008-07-20 19:13:01 -06:00
|
|
|
#endif
|
2008-07-17 01:47:45 -06:00
|
|
|
gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est);
|
|
|
|
if (ops->reset)
|
|
|
|
ops->reset(qdisc);
|
|
|
|
if (ops->destroy)
|
|
|
|
ops->destroy(qdisc);
|
|
|
|
|
|
|
|
module_put(ops->owner);
|
|
|
|
dev_put(qdisc_dev(qdisc));
|
|
|
|
|
2008-10-06 10:54:39 -06:00
|
|
|
kfree_skb(qdisc->gso_skb);
|
2010-03-31 01:06:04 -06:00
|
|
|
/*
|
|
|
|
* gen_estimator est_timer() might access qdisc->q.lock,
|
|
|
|
* wait a RCU grace period before freeing qdisc.
|
|
|
|
*/
|
|
|
|
call_rcu(&qdisc->rcu_head, qdisc_rcu_free);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(qdisc_destroy);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-09-04 00:41:20 -06:00
|
|
|
/* Attach toplevel qdisc to device queue. */
|
|
|
|
struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
|
|
|
|
struct Qdisc *qdisc)
|
|
|
|
{
|
|
|
|
struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
|
|
|
|
spinlock_t *root_lock;
|
|
|
|
|
|
|
|
root_lock = qdisc_lock(oqdisc);
|
|
|
|
spin_lock_bh(root_lock);
|
|
|
|
|
|
|
|
/* Prune old scheduler */
|
|
|
|
if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
|
|
|
|
qdisc_reset(oqdisc);
|
|
|
|
|
|
|
|
/* ... and graft new one */
|
|
|
|
if (qdisc == NULL)
|
|
|
|
qdisc = &noop_qdisc;
|
|
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
|
|
rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
|
|
|
|
|
|
|
|
spin_unlock_bh(root_lock);
|
|
|
|
|
|
|
|
return oqdisc;
|
|
|
|
}
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void attach_one_default_qdisc(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_unused)
|
|
|
|
{
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
|
|
|
if (dev->tx_queue_len) {
|
2010-10-16 07:04:08 -06:00
|
|
|
qdisc = qdisc_create_dflt(dev_queue,
|
2008-07-21 10:56:13 -06:00
|
|
|
&pfifo_fast_ops, TC_H_ROOT);
|
2008-07-17 01:34:19 -06:00
|
|
|
if (!qdisc) {
|
|
|
|
printk(KERN_INFO "%s: activation failed\n", dev->name);
|
|
|
|
return;
|
|
|
|
}
|
2009-08-05 19:44:21 -06:00
|
|
|
|
|
|
|
/* Can by-pass the queue discipline for default qdisc */
|
|
|
|
qdisc->flags |= TCQ_F_CAN_BYPASS;
|
2008-07-17 01:34:19 -06:00
|
|
|
} else {
|
|
|
|
qdisc = &noqueue_qdisc;
|
|
|
|
}
|
|
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
|
|
}
|
|
|
|
|
2009-09-06 02:58:51 -06:00
|
|
|
static void attach_default_qdiscs(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
|
|
|
txq = netdev_get_tx_queue(dev, 0);
|
|
|
|
|
|
|
|
if (!netif_is_multiqueue(dev) || dev->tx_queue_len == 0) {
|
|
|
|
netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
|
|
|
|
dev->qdisc = txq->qdisc_sleeping;
|
|
|
|
atomic_inc(&dev->qdisc->refcnt);
|
|
|
|
} else {
|
2010-10-16 07:04:08 -06:00
|
|
|
qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT);
|
2009-09-06 02:58:51 -06:00
|
|
|
if (qdisc) {
|
|
|
|
qdisc->ops->attach(qdisc);
|
|
|
|
dev->qdisc = qdisc;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void transition_one_qdisc(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_need_watchdog)
|
|
|
|
{
|
2008-07-17 01:53:03 -06:00
|
|
|
struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
|
2008-07-17 01:34:19 -06:00
|
|
|
int *need_watchdog_p = _need_watchdog;
|
|
|
|
|
2008-08-17 22:51:03 -06:00
|
|
|
if (!(new_qdisc->flags & TCQ_F_BUILTIN))
|
|
|
|
clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
|
|
|
|
|
2008-07-17 01:53:03 -06:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
|
2009-05-17 21:55:16 -06:00
|
|
|
if (need_watchdog_p && new_qdisc != &noqueue_qdisc) {
|
|
|
|
dev_queue->trans_start = 0;
|
2008-07-17 01:34:19 -06:00
|
|
|
*need_watchdog_p = 1;
|
2009-05-17 21:55:16 -06:00
|
|
|
}
|
2008-07-17 01:34:19 -06:00
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
void dev_activate(struct net_device *dev)
|
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
int need_watchdog;
|
2008-07-08 18:42:10 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* No queueing discipline is attached to device;
|
2008-07-21 10:56:13 -06:00
|
|
|
create default one i.e. pfifo_fast for devices,
|
|
|
|
which need queueing and noqueue_qdisc for
|
|
|
|
virtual interfaces
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
2009-09-06 02:58:51 -06:00
|
|
|
if (dev->qdisc == &noop_qdisc)
|
|
|
|
attach_default_qdiscs(dev);
|
2009-09-04 00:41:18 -06:00
|
|
|
|
2005-05-03 17:18:52 -06:00
|
|
|
if (!netif_carrier_ok(dev))
|
|
|
|
/* Delay activation until next carrier-on event */
|
|
|
|
return;
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
need_watchdog = 0;
|
|
|
|
netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
|
2010-10-02 00:11:55 -06:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
|
2008-07-17 01:34:19 -06:00
|
|
|
|
|
|
|
if (need_watchdog) {
|
2005-04-16 16:20:36 -06:00
|
|
|
dev->trans_start = jiffies;
|
|
|
|
dev_watchdog_up(dev);
|
|
|
|
}
|
2008-07-08 18:42:10 -06:00
|
|
|
}
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void dev_deactivate_queue(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_qdisc_default)
|
2008-07-08 18:42:10 -06:00
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
2008-07-09 00:10:33 -06:00
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
|
|
|
qdisc = dev_queue->qdisc;
|
2008-07-08 18:42:10 -06:00
|
|
|
if (qdisc) {
|
2008-07-17 01:53:03 -06:00
|
|
|
spin_lock_bh(qdisc_lock(qdisc));
|
|
|
|
|
2008-08-17 22:51:03 -06:00
|
|
|
if (!(qdisc->flags & TCQ_F_BUILTIN))
|
|
|
|
set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
|
|
|
|
|
2008-08-27 03:22:07 -06:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
2008-07-08 18:42:10 -06:00
|
|
|
qdisc_reset(qdisc);
|
2008-07-15 21:14:35 -06:00
|
|
|
|
2008-07-17 01:53:03 -06:00
|
|
|
spin_unlock_bh(qdisc_lock(qdisc));
|
2008-07-08 18:42:10 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-08-17 22:58:07 -06:00
|
|
|
static bool some_qdisc_is_busy(struct net_device *dev)
|
2008-07-17 01:34:19 -06:00
|
|
|
{
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
struct netdev_queue *dev_queue;
|
2008-07-16 02:42:40 -06:00
|
|
|
spinlock_t *root_lock;
|
2008-07-16 01:56:32 -06:00
|
|
|
struct Qdisc *q;
|
2008-07-17 01:34:19 -06:00
|
|
|
int val;
|
|
|
|
|
|
|
|
dev_queue = netdev_get_tx_queue(dev, i);
|
2008-08-13 16:18:38 -06:00
|
|
|
q = dev_queue->qdisc_sleeping;
|
2008-08-02 21:02:43 -06:00
|
|
|
root_lock = qdisc_lock(q);
|
2008-07-17 01:34:19 -06:00
|
|
|
|
2008-08-17 22:58:07 -06:00
|
|
|
spin_lock_bh(root_lock);
|
2008-07-17 01:34:19 -06:00
|
|
|
|
2010-06-02 04:23:51 -06:00
|
|
|
val = (qdisc_is_running(q) ||
|
2008-08-13 16:18:38 -06:00
|
|
|
test_bit(__QDISC_STATE_SCHED, &q->state));
|
2008-07-17 01:34:19 -06:00
|
|
|
|
2008-08-17 22:58:07 -06:00
|
|
|
spin_unlock_bh(root_lock);
|
2008-07-17 01:34:19 -06:00
|
|
|
|
|
|
|
if (val)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2010-12-13 05:44:07 -07:00
|
|
|
void dev_deactivate_many(struct list_head *head)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2010-12-13 05:44:07 -07:00
|
|
|
struct net_device *dev;
|
2007-05-10 15:12:47 -06:00
|
|
|
|
2010-12-13 05:44:07 -07:00
|
|
|
list_for_each_entry(dev, head, unreg_list) {
|
|
|
|
netdev_for_each_tx_queue(dev, dev_deactivate_queue,
|
|
|
|
&noop_qdisc);
|
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
dev_deactivate_queue(dev, dev_ingress_queue(dev),
|
|
|
|
&noop_qdisc);
|
|
|
|
|
|
|
|
dev_watchdog_down(dev);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-18 23:37:58 -06:00
|
|
|
/* Wait for outstanding qdisc-less dev_queue_xmit calls. */
|
2006-06-22 03:28:18 -06:00
|
|
|
synchronize_rcu();
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-06-22 03:28:18 -06:00
|
|
|
/* Wait for outstanding qdisc_run calls. */
|
2010-12-13 05:44:07 -07:00
|
|
|
list_for_each_entry(dev, head, unreg_list)
|
|
|
|
while (some_qdisc_is_busy(dev))
|
|
|
|
yield();
|
|
|
|
}
|
|
|
|
|
|
|
|
void dev_deactivate(struct net_device *dev)
|
|
|
|
{
|
|
|
|
LIST_HEAD(single);
|
|
|
|
|
|
|
|
list_add(&dev->unreg_list, &single);
|
|
|
|
dev_deactivate_many(&single);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-08 18:42:10 -06:00
|
|
|
static void dev_init_scheduler_queue(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
2008-07-17 01:34:19 -06:00
|
|
|
void *_qdisc)
|
2008-07-08 18:42:10 -06:00
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
struct Qdisc *qdisc = _qdisc;
|
|
|
|
|
2008-07-08 18:42:10 -06:00
|
|
|
dev_queue->qdisc = qdisc;
|
|
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
void dev_init_scheduler(struct net_device *dev)
|
|
|
|
{
|
2009-09-04 00:41:18 -06:00
|
|
|
dev->qdisc = &noop_qdisc;
|
2008-07-17 01:34:19 -06:00
|
|
|
netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
|
2010-10-02 00:11:55 -06:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-23 22:20:07 -07:00
|
|
|
setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void shutdown_scheduler_queue(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_qdisc_default)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-07-08 18:42:10 -06:00
|
|
|
struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
|
2008-07-17 01:34:19 -06:00
|
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
2008-07-08 18:42:10 -06:00
|
|
|
|
|
|
|
if (qdisc) {
|
2008-08-27 03:22:07 -06:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
2008-07-08 18:42:10 -06:00
|
|
|
dev_queue->qdisc_sleeping = qdisc_default;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
qdisc_destroy(qdisc);
|
2007-02-09 07:25:16 -07:00
|
|
|
}
|
2008-07-08 18:42:10 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void dev_shutdown(struct net_device *dev)
|
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
|
2010-10-02 00:11:55 -06:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
|
2009-09-04 00:41:18 -06:00
|
|
|
qdisc_destroy(dev->qdisc);
|
|
|
|
dev->qdisc = &noop_qdisc;
|
|
|
|
|
2008-07-25 22:43:18 -06:00
|
|
|
WARN_ON(timer_pending(&dev->watchdog_timer));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|