2019-05-27 00:55:01 -06:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* net/sched/sch_generic.c Generic packet scheduler routines.
|
|
|
|
*
|
|
|
|
* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
|
|
|
|
* Jamal Hadi Salim, <hadi@cyberus.ca> 990601
|
|
|
|
* - Ingress support
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <linux/rtnetlink.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
#include <linux/list.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 02:04:11 -06:00
|
|
|
#include <linux/slab.h>
|
2013-08-03 14:07:47 -06:00
|
|
|
#include <linux/if_vlan.h>
|
2017-12-07 10:58:19 -07:00
|
|
|
#include <linux/skb_array.h>
|
2017-12-06 08:50:28 -07:00
|
|
|
#include <linux/if_macvlan.h>
|
2013-02-11 17:12:03 -07:00
|
|
|
#include <net/sch_generic.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <net/pkt_sched.h>
|
2010-05-11 17:19:48 -06:00
|
|
|
#include <net/dst.h>
|
2017-08-15 13:11:03 -06:00
|
|
|
#include <trace/events/qdisc.h>
|
2019-05-01 20:56:59 -06:00
|
|
|
#include <trace/events/net.h>
|
2017-12-20 02:41:36 -07:00
|
|
|
#include <net/xfrm.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-08-31 11:15:33 -06:00
|
|
|
/* Qdisc to use by default */
|
|
|
|
const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
|
|
|
|
EXPORT_SYMBOL(default_qdisc_ops);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Main transmission queue. */
|
|
|
|
|
2007-04-16 18:02:10 -06:00
|
|
|
/* Modifications to data participating in scheduling must be protected with
|
2008-08-02 21:02:43 -06:00
|
|
|
* qdisc_lock(qdisc) spinlock.
|
2007-04-16 18:02:10 -06:00
|
|
|
*
|
|
|
|
* The idea is the following:
|
2008-07-16 04:22:39 -06:00
|
|
|
* - enqueue, dequeue are serialized via qdisc root lock
|
|
|
|
* - ingress filtering is also serialized via qdisc root lock
|
2007-04-16 18:02:10 -06:00
|
|
|
* - updates to tree and tree walking are only done under the rtnl mutex.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2017-12-07 10:56:23 -07:00
|
|
|
|
|
|
|
static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
|
|
|
|
{
|
|
|
|
const struct netdev_queue *txq = q->dev_queue;
|
|
|
|
spinlock_t *lock = NULL;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
skb = skb_peek(&q->skb_bad_txq);
|
|
|
|
if (skb) {
|
|
|
|
/* check the reason of requeuing without tx lock first */
|
|
|
|
txq = skb_get_tx_queue(txq->dev, skb);
|
|
|
|
if (!netif_xmit_frozen_or_stopped(txq)) {
|
|
|
|
skb = __skb_dequeue(&q->skb_bad_txq);
|
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
qdisc_qstats_cpu_backlog_dec(q, skb);
|
2019-04-10 06:32:41 -06:00
|
|
|
qdisc_qstats_cpu_qlen_dec(q);
|
2017-12-07 10:56:23 -07:00
|
|
|
} else {
|
|
|
|
qdisc_qstats_backlog_dec(q, skb);
|
|
|
|
q->q.qlen--;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
skb = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (lock)
|
|
|
|
spin_unlock(lock);
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb = skb_peek(&q->skb_bad_txq);
|
|
|
|
|
|
|
|
if (unlikely(skb))
|
|
|
|
skb = __skb_dequeue_bad_txq(q);
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
|
|
|
|
struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
spinlock_t *lock = NULL;
|
|
|
|
|
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
__skb_queue_tail(&q->skb_bad_txq, skb);
|
|
|
|
|
2018-03-14 19:53:00 -06:00
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
qdisc_qstats_cpu_backlog_inc(q, skb);
|
2019-04-10 06:32:41 -06:00
|
|
|
qdisc_qstats_cpu_qlen_inc(q);
|
2018-03-14 19:53:00 -06:00
|
|
|
} else {
|
|
|
|
qdisc_qstats_backlog_inc(q, skb);
|
|
|
|
q->q.qlen++;
|
|
|
|
}
|
|
|
|
|
2017-12-07 10:56:23 -07:00
|
|
|
if (lock)
|
|
|
|
spin_unlock(lock);
|
|
|
|
}
|
|
|
|
|
2019-04-10 06:32:39 -06:00
|
|
|
static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
|
2007-06-10 18:31:24 -06:00
|
|
|
{
|
2019-04-10 06:32:39 -06:00
|
|
|
spinlock_t *lock = NULL;
|
2017-12-27 02:05:52 -07:00
|
|
|
|
2019-04-10 06:32:39 -06:00
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
spin_lock(lock);
|
2017-12-27 02:05:52 -07:00
|
|
|
}
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2017-12-27 02:05:52 -07:00
|
|
|
while (skb) {
|
|
|
|
struct sk_buff *next = skb->next;
|
|
|
|
|
|
|
|
__skb_queue_tail(&q->gso_skb, skb);
|
|
|
|
|
2019-04-10 06:32:39 -06:00
|
|
|
/* it's still part of the queue */
|
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
qdisc_qstats_cpu_requeues_inc(q);
|
|
|
|
qdisc_qstats_cpu_backlog_inc(q, skb);
|
2019-04-10 06:32:41 -06:00
|
|
|
qdisc_qstats_cpu_qlen_inc(q);
|
2019-04-10 06:32:39 -06:00
|
|
|
} else {
|
|
|
|
q->qstats.requeues++;
|
|
|
|
qdisc_qstats_backlog_inc(q, skb);
|
|
|
|
q->q.qlen++;
|
|
|
|
}
|
2017-12-27 02:05:52 -07:00
|
|
|
|
|
|
|
skb = next;
|
|
|
|
}
|
2019-04-10 06:32:39 -06:00
|
|
|
if (lock)
|
|
|
|
spin_unlock(lock);
|
2017-12-07 10:55:45 -07:00
|
|
|
__netif_schedule(q);
|
|
|
|
}
|
|
|
|
|
2014-10-03 16:31:07 -06:00
|
|
|
static void try_bulk_dequeue_skb(struct Qdisc *q,
|
|
|
|
struct sk_buff *skb,
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
const struct netdev_queue *txq,
|
|
|
|
int *packets)
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 14:35:59 -06:00
|
|
|
{
|
2014-10-03 16:31:07 -06:00
|
|
|
int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 14:35:59 -06:00
|
|
|
|
|
|
|
while (bytelimit > 0) {
|
2014-10-03 16:31:07 -06:00
|
|
|
struct sk_buff *nskb = q->dequeue(q);
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 14:35:59 -06:00
|
|
|
|
2014-10-03 16:31:07 -06:00
|
|
|
if (!nskb)
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 14:35:59 -06:00
|
|
|
break;
|
|
|
|
|
2014-10-03 16:31:07 -06:00
|
|
|
bytelimit -= nskb->len; /* covers GSO len */
|
|
|
|
skb->next = nskb;
|
|
|
|
skb = nskb;
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
(*packets)++; /* GSO counts as one pkt */
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 14:35:59 -06:00
|
|
|
}
|
2018-07-29 21:42:53 -06:00
|
|
|
skb_mark_not_on_list(skb);
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 14:35:59 -06:00
|
|
|
}
|
|
|
|
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
/* This variant of try_bulk_dequeue_skb() makes sure
|
|
|
|
* all skbs in the chain are for the same txq
|
|
|
|
*/
|
|
|
|
static void try_bulk_dequeue_skb_slow(struct Qdisc *q,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
int *packets)
|
|
|
|
{
|
|
|
|
int mapping = skb_get_queue_mapping(skb);
|
|
|
|
struct sk_buff *nskb;
|
|
|
|
int cnt = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
nskb = q->dequeue(q);
|
|
|
|
if (!nskb)
|
|
|
|
break;
|
|
|
|
if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
|
2017-12-07 10:56:23 -07:00
|
|
|
qdisc_enqueue_skb_bad_txq(q, nskb);
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
skb->next = nskb;
|
|
|
|
skb = nskb;
|
|
|
|
} while (++cnt < 8);
|
|
|
|
(*packets) += cnt;
|
2018-07-29 21:42:53 -06:00
|
|
|
skb_mark_not_on_list(skb);
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
}
|
|
|
|
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 14:35:59 -06:00
|
|
|
/* Note that dequeue_skb can possibly return a SKB list (via skb->next).
|
|
|
|
* A requeued skb (via q->gso_skb) can also be a SKB list.
|
|
|
|
*/
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
|
|
|
|
int *packets)
|
2007-06-10 18:31:24 -06:00
|
|
|
{
|
2012-12-11 08:54:33 -07:00
|
|
|
const struct netdev_queue *txq = q->dev_queue;
|
2017-12-07 10:56:42 -07:00
|
|
|
struct sk_buff *skb = NULL;
|
2008-10-06 10:54:39 -06:00
|
|
|
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
*packets = 1;
|
2017-12-07 10:55:45 -07:00
|
|
|
if (unlikely(!skb_queue_empty(&q->gso_skb))) {
|
|
|
|
spinlock_t *lock = NULL;
|
|
|
|
|
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
skb = skb_peek(&q->gso_skb);
|
|
|
|
|
|
|
|
/* skb may be null if another cpu pulls gso_skb off in between
|
|
|
|
* empty check and lock.
|
|
|
|
*/
|
|
|
|
if (!skb) {
|
|
|
|
if (lock)
|
|
|
|
spin_unlock(lock);
|
|
|
|
goto validate;
|
|
|
|
}
|
|
|
|
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
/* skb in gso_skb were already validated */
|
|
|
|
*validate = false;
|
2017-12-20 02:41:36 -07:00
|
|
|
if (xfrm_offload(skb))
|
|
|
|
*validate = true;
|
2008-09-22 23:16:23 -06:00
|
|
|
/* check the reason of requeuing without tx lock first */
|
2014-08-27 03:11:27 -06:00
|
|
|
txq = skb_get_tx_queue(txq->dev, skb);
|
2011-11-28 09:32:44 -07:00
|
|
|
if (!netif_xmit_frozen_or_stopped(txq)) {
|
2017-12-07 10:55:45 -07:00
|
|
|
skb = __skb_dequeue(&q->gso_skb);
|
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
qdisc_qstats_cpu_backlog_dec(q, skb);
|
2019-04-10 06:32:41 -06:00
|
|
|
qdisc_qstats_cpu_qlen_dec(q);
|
2017-12-07 10:55:45 -07:00
|
|
|
} else {
|
|
|
|
qdisc_qstats_backlog_dec(q, skb);
|
|
|
|
q->q.qlen--;
|
|
|
|
}
|
|
|
|
} else {
|
2008-09-22 23:16:23 -06:00
|
|
|
skb = NULL;
|
2017-12-07 10:55:45 -07:00
|
|
|
}
|
|
|
|
if (lock)
|
|
|
|
spin_unlock(lock);
|
2017-08-15 13:11:03 -06:00
|
|
|
goto trace;
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
}
|
2017-12-07 10:55:45 -07:00
|
|
|
validate:
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
*validate = true;
|
2017-12-07 10:56:42 -07:00
|
|
|
|
|
|
|
if ((q->flags & TCQ_F_ONETXQUEUE) &&
|
|
|
|
netif_xmit_frozen_or_stopped(txq))
|
|
|
|
return skb;
|
|
|
|
|
2017-12-07 10:56:23 -07:00
|
|
|
skb = qdisc_dequeue_skb_bad_txq(q);
|
|
|
|
if (unlikely(skb))
|
|
|
|
goto bulk;
|
2017-12-07 10:56:42 -07:00
|
|
|
skb = q->dequeue(q);
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
if (skb) {
|
|
|
|
bulk:
|
|
|
|
if (qdisc_may_bulk(q))
|
|
|
|
try_bulk_dequeue_skb(q, skb, txq, packets);
|
|
|
|
else
|
|
|
|
try_bulk_dequeue_skb_slow(q, skb, packets);
|
2008-09-22 23:16:23 -06:00
|
|
|
}
|
2017-08-15 13:11:03 -06:00
|
|
|
trace:
|
|
|
|
trace_qdisc_dequeue(q, txq, *packets, skb);
|
2007-06-10 18:31:24 -06:00
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2007-02-09 07:25:16 -07:00
|
|
|
/*
|
2014-09-02 08:35:33 -06:00
|
|
|
* Transmit possibly several skbs, and handle the return status as
|
2016-06-06 10:37:15 -06:00
|
|
|
* required. Owning running seqcount bit guarantees that
|
2014-09-02 08:35:33 -06:00
|
|
|
* only one CPU can execute this function.
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
*
|
|
|
|
* Returns to the caller:
|
2017-12-07 10:54:47 -07:00
|
|
|
* false - hardware queue frozen backoff
|
|
|
|
* true - feel free to send more pkts
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
*/
|
2017-12-07 10:54:47 -07:00
|
|
|
bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
|
|
|
|
struct net_device *dev, struct netdev_queue *txq,
|
|
|
|
spinlock_t *root_lock, bool validate)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-11-13 21:40:55 -07:00
|
|
|
int ret = NETDEV_TX_BUSY;
|
2017-12-20 02:41:36 -07:00
|
|
|
bool again = false;
|
2008-07-16 02:42:40 -06:00
|
|
|
|
|
|
|
/* And release qdisc */
|
2017-12-07 10:54:25 -07:00
|
|
|
if (root_lock)
|
|
|
|
spin_unlock(root_lock);
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2014-10-03 16:31:07 -06:00
|
|
|
/* Note that we validate skb (GSO, checksum, ...) outside of locks */
|
|
|
|
if (validate)
|
2017-12-20 02:41:36 -07:00
|
|
|
skb = validate_xmit_skb_list(skb, dev, &again);
|
|
|
|
|
|
|
|
#ifdef CONFIG_XFRM_OFFLOAD
|
|
|
|
if (unlikely(again)) {
|
|
|
|
if (root_lock)
|
|
|
|
spin_lock(root_lock);
|
|
|
|
|
|
|
|
dev_requeue_skb(skb, q);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#endif
|
2009-11-09 23:14:14 -07:00
|
|
|
|
2016-04-12 00:45:52 -06:00
|
|
|
if (likely(skb)) {
|
2014-10-03 16:31:07 -06:00
|
|
|
HARD_TX_LOCK(dev, txq, smp_processor_id());
|
|
|
|
if (!netif_xmit_frozen_or_stopped(txq))
|
|
|
|
skb = dev_hard_start_xmit(skb, dev, txq, &ret);
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2014-10-03 16:31:07 -06:00
|
|
|
HARD_TX_UNLOCK(dev, txq);
|
2016-04-12 00:45:52 -06:00
|
|
|
} else {
|
2017-12-07 10:54:25 -07:00
|
|
|
if (root_lock)
|
|
|
|
spin_lock(root_lock);
|
2017-12-07 10:54:47 -07:00
|
|
|
return true;
|
2014-10-03 16:31:07 -06:00
|
|
|
}
|
2017-12-07 10:54:25 -07:00
|
|
|
|
|
|
|
if (root_lock)
|
|
|
|
spin_lock(root_lock);
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2017-12-07 10:54:47 -07:00
|
|
|
if (!dev_xmit_complete(ret)) {
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
/* Driver returned NETDEV_TX_BUSY - requeue skb */
|
2012-05-13 15:56:26 -06:00
|
|
|
if (unlikely(ret != NETDEV_TX_BUSY))
|
|
|
|
net_warn_ratelimited("BUG %s code %d qlen %d\n",
|
|
|
|
dev->name, ret, q->q.qlen);
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
|
2017-12-07 10:54:47 -07:00
|
|
|
dev_requeue_skb(skb, q);
|
|
|
|
return false;
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 20:56:09 -06:00
|
|
|
}
|
2007-06-10 18:31:24 -06:00
|
|
|
|
2017-12-07 10:54:47 -07:00
|
|
|
return true;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2009-08-05 19:44:21 -06:00
|
|
|
/*
|
|
|
|
* NOTE: Called under qdisc_lock(q) with locally disabled BH.
|
|
|
|
*
|
2016-06-06 10:37:15 -06:00
|
|
|
* running seqcount guarantees only one CPU can process
|
2009-08-05 19:44:21 -06:00
|
|
|
* this qdisc at a time. qdisc_lock(q) serializes queue accesses for
|
|
|
|
* this queue.
|
|
|
|
*
|
|
|
|
* netif_tx_lock serializes accesses to device driver.
|
|
|
|
*
|
|
|
|
* qdisc_lock(q) and netif_tx_lock are mutually exclusive,
|
|
|
|
* if one is grabbed, another must be free.
|
|
|
|
*
|
|
|
|
* Note, that this procedure can be called by a watchdog timer
|
|
|
|
*
|
|
|
|
* Returns to the caller:
|
|
|
|
* 0 - queue is empty or throttled.
|
|
|
|
* >0 - queue is not empty.
|
|
|
|
*
|
|
|
|
*/
|
2017-12-07 10:54:47 -07:00
|
|
|
static inline bool qdisc_restart(struct Qdisc *q, int *packets)
|
2009-08-05 19:44:21 -06:00
|
|
|
{
|
2017-12-07 10:54:25 -07:00
|
|
|
spinlock_t *root_lock = NULL;
|
2009-08-05 19:44:21 -06:00
|
|
|
struct netdev_queue *txq;
|
|
|
|
struct net_device *dev;
|
|
|
|
struct sk_buff *skb;
|
2018-05-15 02:50:31 -06:00
|
|
|
bool validate;
|
2009-08-05 19:44:21 -06:00
|
|
|
|
|
|
|
/* Dequeue packet */
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
skb = dequeue_skb(q, &validate, packets);
|
2018-05-15 02:50:31 -06:00
|
|
|
if (unlikely(!skb))
|
2017-12-07 10:54:47 -07:00
|
|
|
return false;
|
2014-08-27 03:11:27 -06:00
|
|
|
|
2018-05-15 02:50:31 -06:00
|
|
|
if (!(q->flags & TCQ_F_NOLOCK))
|
2017-12-07 10:54:25 -07:00
|
|
|
root_lock = qdisc_lock(q);
|
|
|
|
|
2009-08-05 19:44:21 -06:00
|
|
|
dev = qdisc_dev(q);
|
2014-08-27 03:11:27 -06:00
|
|
|
txq = skb_get_tx_queue(dev, skb);
|
2009-08-05 19:44:21 -06:00
|
|
|
|
2018-05-15 02:50:31 -06:00
|
|
|
return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
|
2009-08-05 19:44:21 -06:00
|
|
|
}
|
|
|
|
|
2008-07-16 03:15:04 -06:00
|
|
|
void __qdisc_run(struct Qdisc *q)
|
2006-06-20 00:57:59 -06:00
|
|
|
{
|
2016-12-29 13:37:21 -07:00
|
|
|
int quota = dev_tx_weight;
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
int packets;
|
2008-03-28 17:25:26 -06:00
|
|
|
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
while (qdisc_restart(q, &packets)) {
|
2008-03-28 17:25:26 -06:00
|
|
|
/*
|
2011-06-26 02:13:54 -06:00
|
|
|
* Ordered by possible occurrence: Postpone processing if
|
|
|
|
* 1. we've exceeded packet quota
|
|
|
|
* 2. another process needs the CPU;
|
2008-03-28 17:25:26 -06:00
|
|
|
*/
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 04:18:10 -06:00
|
|
|
quota -= packets;
|
|
|
|
if (quota <= 0 || need_resched()) {
|
2008-07-16 03:15:04 -06:00
|
|
|
__netif_schedule(q);
|
2007-05-10 05:55:14 -06:00
|
|
|
break;
|
2008-03-28 17:25:26 -06:00
|
|
|
}
|
|
|
|
}
|
2006-06-20 00:57:59 -06:00
|
|
|
}
|
|
|
|
|
2009-05-17 21:55:16 -06:00
|
|
|
unsigned long dev_trans_start(struct net_device *dev)
|
|
|
|
{
|
2013-08-03 14:07:47 -06:00
|
|
|
unsigned long val, res;
|
2009-05-17 21:55:16 -06:00
|
|
|
unsigned int i;
|
|
|
|
|
2013-08-03 14:07:47 -06:00
|
|
|
if (is_vlan_dev(dev))
|
|
|
|
dev = vlan_dev_real_dev(dev);
|
2017-12-06 08:50:28 -07:00
|
|
|
else if (netif_is_macvlan(dev))
|
|
|
|
dev = macvlan_dev_real_dev(dev);
|
2016-05-03 08:33:14 -06:00
|
|
|
res = netdev_get_tx_queue(dev, 0)->trans_start;
|
|
|
|
for (i = 1; i < dev->num_tx_queues; i++) {
|
2009-05-17 21:55:16 -06:00
|
|
|
val = netdev_get_tx_queue(dev, i)->trans_start;
|
|
|
|
if (val && time_after(val, res))
|
|
|
|
res = val;
|
|
|
|
}
|
2013-08-03 14:07:47 -06:00
|
|
|
|
2009-05-17 21:55:16 -06:00
|
|
|
return res;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dev_trans_start);
|
|
|
|
|
2017-10-16 18:29:17 -06:00
|
|
|
static void dev_watchdog(struct timer_list *t)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2017-10-16 18:29:17 -06:00
|
|
|
struct net_device *dev = from_timer(dev, t, watchdog_timer);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_lock(dev);
|
2008-07-17 01:34:19 -06:00
|
|
|
if (!qdisc_tx_is_noop(dev)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (netif_device_present(dev) &&
|
|
|
|
netif_running(dev) &&
|
|
|
|
netif_carrier_ok(dev)) {
|
2009-05-17 21:55:16 -06:00
|
|
|
int some_queue_timedout = 0;
|
2008-07-17 01:34:19 -06:00
|
|
|
unsigned int i;
|
2009-05-17 21:55:16 -06:00
|
|
|
unsigned long trans_start;
|
2008-07-17 01:34:19 -06:00
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
|
|
|
|
txq = netdev_get_tx_queue(dev, i);
|
2016-05-03 08:33:14 -06:00
|
|
|
trans_start = txq->trans_start;
|
2011-11-28 09:32:44 -07:00
|
|
|
if (netif_xmit_stopped(txq) &&
|
2009-05-17 21:55:16 -06:00
|
|
|
time_after(jiffies, (trans_start +
|
|
|
|
dev->watchdog_timeo))) {
|
|
|
|
some_queue_timedout = 1;
|
2011-11-16 05:15:10 -07:00
|
|
|
txq->trans_timeout++;
|
2008-07-17 01:34:19 -06:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2006-05-16 16:02:12 -06:00
|
|
|
|
2009-05-17 21:55:16 -06:00
|
|
|
if (some_queue_timedout) {
|
2019-05-01 20:56:59 -06:00
|
|
|
trace_net_dev_xmit_timeout(dev, i);
|
2009-05-17 21:55:16 -06:00
|
|
|
WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
|
2011-06-06 17:41:33 -06:00
|
|
|
dev->name, netdev_drivername(dev), i);
|
2008-11-19 22:32:24 -07:00
|
|
|
dev->netdev_ops->ndo_tx_timeout(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-07-17 01:34:19 -06:00
|
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
|
|
round_jiffies(jiffies +
|
|
|
|
dev->watchdog_timeo)))
|
2005-04-16 16:20:36 -06:00
|
|
|
dev_hold(dev);
|
|
|
|
}
|
|
|
|
}
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_unlock(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
dev_put(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __netdev_watchdog_up(struct net_device *dev)
|
|
|
|
{
|
2008-11-19 22:32:24 -07:00
|
|
|
if (dev->netdev_ops->ndo_tx_timeout) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (dev->watchdog_timeo <= 0)
|
|
|
|
dev->watchdog_timeo = 5*HZ;
|
2007-05-31 22:28:44 -06:00
|
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
|
|
round_jiffies(jiffies + dev->watchdog_timeo)))
|
2005-04-16 16:20:36 -06:00
|
|
|
dev_hold(dev);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dev_watchdog_up(struct net_device *dev)
|
|
|
|
{
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dev_watchdog_down(struct net_device *dev)
|
|
|
|
{
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_lock_bh(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (del_timer(&dev->watchdog_timer))
|
2006-03-20 23:32:28 -07:00
|
|
|
dev_put(dev);
|
2006-06-09 13:20:56 -06:00
|
|
|
netif_tx_unlock_bh(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 17:41:36 -06:00
|
|
|
/**
|
|
|
|
* netif_carrier_on - set carrier
|
|
|
|
* @dev: network device
|
|
|
|
*
|
2019-02-07 09:14:32 -07:00
|
|
|
* Device has detected acquisition of carrier.
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 17:41:36 -06:00
|
|
|
*/
|
2005-08-11 16:32:53 -06:00
|
|
|
void netif_carrier_on(struct net_device *dev)
|
|
|
|
{
|
2007-10-18 00:26:43 -06:00
|
|
|
if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
2008-11-19 16:33:54 -07:00
|
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
|
|
return;
|
2018-01-18 10:59:13 -07:00
|
|
|
atomic_inc(&dev->carrier_up_count);
|
2005-08-11 16:32:53 -06:00
|
|
|
linkwatch_fire_event(dev);
|
2007-10-18 00:26:43 -06:00
|
|
|
if (netif_running(dev))
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
}
|
2005-08-11 16:32:53 -06:00
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(netif_carrier_on);
|
2005-08-11 16:32:53 -06:00
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 17:41:36 -06:00
|
|
|
/**
|
|
|
|
* netif_carrier_off - clear carrier
|
|
|
|
* @dev: network device
|
|
|
|
*
|
|
|
|
* Device has detected loss of carrier.
|
|
|
|
*/
|
2005-08-11 16:32:53 -06:00
|
|
|
void netif_carrier_off(struct net_device *dev)
|
|
|
|
{
|
2008-11-19 16:33:54 -07:00
|
|
|
if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
|
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
|
|
return;
|
2018-01-18 10:59:13 -07:00
|
|
|
atomic_inc(&dev->carrier_down_count);
|
2005-08-11 16:32:53 -06:00
|
|
|
linkwatch_fire_event(dev);
|
2008-11-19 16:33:54 -07:00
|
|
|
}
|
2005-08-11 16:32:53 -06:00
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(netif_carrier_off);
|
2005-08-11 16:32:53 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
|
|
|
|
under all circumstances. It is difficult to invent anything faster or
|
|
|
|
cheaper.
|
|
|
|
*/
|
|
|
|
|
2016-06-22 00:16:49 -06:00
|
|
|
static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
|
|
|
|
struct sk_buff **to_free)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2016-06-22 00:16:49 -06:00
|
|
|
__qdisc_drop(skb, to_free);
|
2005-04-16 16:20:36 -06:00
|
|
|
return NET_XMIT_CN;
|
|
|
|
}
|
|
|
|
|
2013-12-10 05:55:31 -07:00
|
|
|
static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2007-11-14 02:44:41 -07:00
|
|
|
struct Qdisc_ops noop_qdisc_ops __read_mostly = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.id = "noop",
|
|
|
|
.priv_size = 0,
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
2008-10-31 01:45:27 -06:00
|
|
|
.peek = noop_dequeue,
|
2005-04-16 16:20:36 -06:00
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
2008-07-16 02:42:40 -06:00
|
|
|
static struct netdev_queue noop_netdev_queue = {
|
2019-02-24 19:43:06 -07:00
|
|
|
RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
|
2008-10-20 00:37:47 -06:00
|
|
|
.qdisc_sleeping = &noop_qdisc,
|
2008-07-16 02:42:40 -06:00
|
|
|
};
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
struct Qdisc noop_qdisc = {
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
.flags = TCQ_F_BUILTIN,
|
2007-02-09 07:25:16 -07:00
|
|
|
.ops = &noop_qdisc_ops,
|
2008-07-17 01:53:03 -06:00
|
|
|
.q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
|
2008-07-16 02:42:40 -06:00
|
|
|
.dev_queue = &noop_netdev_queue,
|
2016-06-06 10:37:15 -06:00
|
|
|
.running = SEQCNT_ZERO(noop_qdisc.running),
|
2010-10-15 13:22:34 -06:00
|
|
|
.busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
|
2018-10-09 16:20:50 -06:00
|
|
|
.gso_skb = {
|
|
|
|
.next = (struct sk_buff *)&noop_qdisc.gso_skb,
|
|
|
|
.prev = (struct sk_buff *)&noop_qdisc.gso_skb,
|
|
|
|
.qlen = 0,
|
|
|
|
.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock),
|
|
|
|
},
|
|
|
|
.skb_bad_txq = {
|
|
|
|
.next = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
|
|
|
|
.prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
|
|
|
|
.qlen = 0,
|
|
|
|
.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock),
|
|
|
|
},
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(noop_qdisc);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2017-12-20 10:35:13 -07:00
|
|
|
static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
|
|
|
|
struct netlink_ext_ack *extack)
|
2015-08-27 13:21:38 -06:00
|
|
|
{
|
|
|
|
/* register_qdisc() assigns a default of noop_enqueue if unset,
|
|
|
|
* but __dev_queue_xmit() treats noqueue only as such
|
|
|
|
* if this is NULL - so clear it here. */
|
|
|
|
qdisc->enqueue = NULL;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.id = "noqueue",
|
|
|
|
.priv_size = 0,
|
2015-08-27 13:21:38 -06:00
|
|
|
.init = noqueue_init,
|
2005-04-16 16:20:36 -06:00
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
2008-10-31 01:45:27 -06:00
|
|
|
.peek = noop_dequeue,
|
2005-04-16 16:20:36 -06:00
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
2011-01-19 12:26:56 -07:00
|
|
|
static const u8 prio2band[TC_PRIO_MAX + 1] = {
|
|
|
|
1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1
|
|
|
|
};
|
2008-07-21 10:56:13 -06:00
|
|
|
|
|
|
|
/* 3-band FIFO queue: old style, but should be a bit faster than
|
|
|
|
generic prio+fifo combination.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define PFIFO_FAST_BANDS 3
|
|
|
|
|
2009-08-18 15:55:59 -06:00
|
|
|
/*
|
|
|
|
* Private data for a pfifo_fast scheduler containing:
|
2017-12-07 10:58:19 -07:00
|
|
|
* - rings for priority bands
|
2009-08-18 15:55:59 -06:00
|
|
|
*/
|
|
|
|
struct pfifo_fast_priv {
|
2017-12-07 10:58:19 -07:00
|
|
|
struct skb_array q[PFIFO_FAST_BANDS];
|
2009-08-18 15:55:59 -06:00
|
|
|
};
|
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
|
|
|
|
int band)
|
2008-07-21 10:56:13 -06:00
|
|
|
{
|
2017-12-07 10:58:19 -07:00
|
|
|
return &priv->q[band];
|
2008-07-21 10:56:13 -06:00
|
|
|
}
|
|
|
|
|
2016-06-22 00:16:49 -06:00
|
|
|
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
|
|
|
|
struct sk_buff **to_free)
|
2005-06-18 23:58:35 -06:00
|
|
|
{
|
2017-12-07 10:58:19 -07:00
|
|
|
int band = prio2band[skb->priority & TC_PRIO_MAX];
|
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
2018-03-14 19:53:00 -06:00
|
|
|
unsigned int pkt_len = qdisc_pkt_len(skb);
|
2017-12-07 10:58:19 -07:00
|
|
|
int err;
|
2005-06-18 23:58:15 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
err = skb_array_produce(q, skb);
|
|
|
|
|
|
|
|
if (unlikely(err))
|
|
|
|
return qdisc_drop_cpu(skb, qdisc, to_free);
|
|
|
|
|
2019-04-10 06:32:40 -06:00
|
|
|
qdisc_update_stats_at_enqueue(qdisc, pkt_len);
|
2017-12-07 10:58:19 -07:00
|
|
|
return NET_XMIT_SUCCESS;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2011-01-19 12:26:56 -07:00
|
|
|
static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2017-12-07 10:58:19 -07:00
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
int band;
|
2016-09-17 16:57:32 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
2009-08-18 15:55:59 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
if (__skb_array_empty(q))
|
|
|
|
continue;
|
2009-08-18 15:55:59 -06:00
|
|
|
|
2018-05-15 08:24:37 -06:00
|
|
|
skb = __skb_array_consume(q);
|
2017-12-07 10:58:19 -07:00
|
|
|
}
|
|
|
|
if (likely(skb)) {
|
2019-04-10 06:32:40 -06:00
|
|
|
qdisc_update_stats_at_dequeue(qdisc, skb);
|
2019-03-22 09:01:55 -06:00
|
|
|
} else {
|
|
|
|
qdisc->empty = true;
|
2008-07-21 10:56:13 -06:00
|
|
|
}
|
2005-06-18 23:58:53 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
return skb;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2011-01-19 12:26:56 -07:00
|
|
|
static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
|
2008-10-31 01:45:27 -06:00
|
|
|
{
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2017-12-07 10:58:19 -07:00
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
int band;
|
2009-08-18 15:55:59 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
2008-10-31 01:45:27 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
skb = __skb_array_peek(q);
|
2008-10-31 01:45:27 -06:00
|
|
|
}
|
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
return skb;
|
2008-10-31 01:45:27 -06:00
|
|
|
}
|
|
|
|
|
2011-01-19 12:26:56 -07:00
|
|
|
static void pfifo_fast_reset(struct Qdisc *qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2017-12-07 10:58:19 -07:00
|
|
|
int i, band;
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2008-07-21 10:56:13 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
for (band = 0; band < PFIFO_FAST_BANDS; band++) {
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
|
|
|
struct sk_buff *skb;
|
2008-07-21 10:56:13 -06:00
|
|
|
|
2017-12-18 15:34:26 -07:00
|
|
|
/* NULL ring is possible if destroy path is due to a failed
|
|
|
|
* skb_array_init() in pfifo_fast_init() case.
|
|
|
|
*/
|
|
|
|
if (!q->ring.queue)
|
|
|
|
continue;
|
|
|
|
|
2018-05-15 08:24:37 -06:00
|
|
|
while ((skb = __skb_array_consume(q)) != NULL)
|
2017-12-07 10:58:19 -07:00
|
|
|
kfree_skb(skb);
|
|
|
|
}
|
|
|
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
struct gnet_stats_queue *q = per_cpu_ptr(qdisc->cpu_qstats, i);
|
|
|
|
|
|
|
|
q->backlog = 0;
|
2019-04-10 06:32:41 -06:00
|
|
|
q->qlen = 0;
|
2017-12-07 10:58:19 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-21 10:56:13 -06:00
|
|
|
static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
|
|
|
|
|
2011-01-19 12:26:56 -07:00
|
|
|
memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
|
2012-03-29 03:11:39 -06:00
|
|
|
if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
|
|
|
|
goto nla_put_failure;
|
2008-07-21 10:56:13 -06:00
|
|
|
return skb->len;
|
|
|
|
|
|
|
|
nla_put_failure:
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2017-12-20 10:35:13 -07:00
|
|
|
static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
|
|
|
|
struct netlink_ext_ack *extack)
|
2008-07-21 10:56:13 -06:00
|
|
|
{
|
2017-12-07 10:58:19 -07:00
|
|
|
unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
|
2009-08-18 15:55:59 -06:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2017-12-07 10:58:19 -07:00
|
|
|
int prio;
|
|
|
|
|
|
|
|
/* guard against zero length rings */
|
|
|
|
if (!qlen)
|
|
|
|
return -EINVAL;
|
2008-07-21 10:56:13 -06:00
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
|
|
|
|
struct skb_array *q = band2list(priv, prio);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = skb_array_init(q, qlen, GFP_KERNEL);
|
|
|
|
if (err)
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2008-07-21 10:56:13 -06:00
|
|
|
|
2011-01-21 17:26:09 -07:00
|
|
|
/* Can by-pass the queue discipline */
|
|
|
|
qdisc->flags |= TCQ_F_CAN_BYPASS;
|
2008-07-21 10:56:13 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-12-07 10:58:19 -07:00
|
|
|
static void pfifo_fast_destroy(struct Qdisc *sch)
|
|
|
|
{
|
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(sch);
|
|
|
|
int prio;
|
|
|
|
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
|
|
|
|
struct skb_array *q = band2list(priv, prio);
|
|
|
|
|
|
|
|
/* NULL ring is possible if destroy path is due to a failed
|
|
|
|
* skb_array_init() in pfifo_fast_init() case.
|
|
|
|
*/
|
2017-12-18 15:34:26 -07:00
|
|
|
if (!q->ring.queue)
|
2017-12-07 10:58:19 -07:00
|
|
|
continue;
|
|
|
|
/* Destroy ring but no need to kfree_skb because a call to
|
|
|
|
* pfifo_fast_reset() has already done that work.
|
|
|
|
*/
|
|
|
|
ptr_ring_cleanup(&q->ring, NULL);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-25 19:26:24 -07:00
|
|
|
static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch,
|
|
|
|
unsigned int new_len)
|
|
|
|
{
|
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(sch);
|
|
|
|
struct skb_array *bands[PFIFO_FAST_BANDS];
|
|
|
|
int prio;
|
|
|
|
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
|
|
|
|
struct skb_array *q = band2list(priv, prio);
|
|
|
|
|
|
|
|
bands[prio] = q;
|
|
|
|
}
|
|
|
|
|
|
|
|
return skb_array_resize_multiple(bands, PFIFO_FAST_BANDS, new_len,
|
|
|
|
GFP_KERNEL);
|
|
|
|
}
|
|
|
|
|
2009-09-06 02:58:51 -06:00
|
|
|
struct Qdisc_ops pfifo_fast_ops __read_mostly = {
|
2008-07-21 10:56:13 -06:00
|
|
|
.id = "pfifo_fast",
|
2009-08-18 15:55:59 -06:00
|
|
|
.priv_size = sizeof(struct pfifo_fast_priv),
|
2008-07-21 10:56:13 -06:00
|
|
|
.enqueue = pfifo_fast_enqueue,
|
|
|
|
.dequeue = pfifo_fast_dequeue,
|
2008-10-31 01:45:27 -06:00
|
|
|
.peek = pfifo_fast_peek,
|
2008-07-21 10:56:13 -06:00
|
|
|
.init = pfifo_fast_init,
|
2017-12-07 10:58:19 -07:00
|
|
|
.destroy = pfifo_fast_destroy,
|
2008-07-21 10:56:13 -06:00
|
|
|
.reset = pfifo_fast_reset,
|
|
|
|
.dump = pfifo_fast_dump,
|
2018-01-25 19:26:24 -07:00
|
|
|
.change_tx_queue_len = pfifo_fast_change_tx_queue_len,
|
2005-04-16 16:20:36 -06:00
|
|
|
.owner = THIS_MODULE,
|
2017-12-07 10:58:19 -07:00
|
|
|
.static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
2016-03-02 09:21:43 -07:00
|
|
|
EXPORT_SYMBOL(pfifo_fast_ops);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-09-04 19:02:56 -06:00
|
|
|
static struct lock_class_key qdisc_tx_busylock;
|
2016-06-06 10:37:15 -06:00
|
|
|
static struct lock_class_key qdisc_running_key;
|
2012-09-04 19:02:56 -06:00
|
|
|
|
2008-07-08 18:06:30 -06:00
|
|
|
struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
|
2017-12-20 10:35:20 -07:00
|
|
|
const struct Qdisc_ops *ops,
|
|
|
|
struct netlink_ext_ack *extack)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
struct Qdisc *sch;
|
2011-03-03 12:10:02 -07:00
|
|
|
unsigned int size = QDISC_ALIGN(sizeof(*sch)) + ops->priv_size;
|
2005-07-05 15:15:09 -06:00
|
|
|
int err = -ENOBUFS;
|
2017-10-16 19:01:23 -06:00
|
|
|
struct net_device *dev;
|
|
|
|
|
|
|
|
if (!dev_queue) {
|
2017-12-20 10:35:20 -07:00
|
|
|
NL_SET_ERR_MSG(extack, "No device queue given");
|
2017-10-16 19:01:23 -06:00
|
|
|
err = -EINVAL;
|
|
|
|
goto errout;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2017-10-16 19:01:23 -06:00
|
|
|
dev = dev_queue->dev;
|
2010-11-29 01:14:37 -07:00
|
|
|
p = kzalloc_node(size, GFP_KERNEL,
|
|
|
|
netdev_queue_numa_node_read(dev_queue));
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!p)
|
2005-07-05 15:15:09 -06:00
|
|
|
goto errout;
|
|
|
|
sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
|
2011-03-03 12:10:02 -07:00
|
|
|
/* if we got non aligned memory, ask more and do alignment ourself */
|
|
|
|
if (sch != p) {
|
|
|
|
kfree(p);
|
|
|
|
p = kzalloc_node(size + QDISC_ALIGNTO - 1, GFP_KERNEL,
|
|
|
|
netdev_queue_numa_node_read(dev_queue));
|
|
|
|
if (!p)
|
|
|
|
goto errout;
|
|
|
|
sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
|
|
|
|
sch->padded = (char *) sch - (char *) p;
|
|
|
|
}
|
2017-12-07 10:55:45 -07:00
|
|
|
__skb_queue_head_init(&sch->gso_skb);
|
2017-12-07 10:56:23 -07:00
|
|
|
__skb_queue_head_init(&sch->skb_bad_txq);
|
2016-09-17 16:57:34 -06:00
|
|
|
qdisc_skb_head_init(&sch->q);
|
|
|
|
spin_lock_init(&sch->q.lock);
|
2012-09-04 19:02:56 -06:00
|
|
|
|
2017-12-07 10:55:26 -07:00
|
|
|
if (ops->static_flags & TCQ_F_CPUSTATS) {
|
|
|
|
sch->cpu_bstats =
|
|
|
|
netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
|
|
|
|
if (!sch->cpu_bstats)
|
|
|
|
goto errout1;
|
|
|
|
|
|
|
|
sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
|
|
|
|
if (!sch->cpu_qstats) {
|
|
|
|
free_percpu(sch->cpu_bstats);
|
|
|
|
goto errout1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-06-02 06:09:29 -06:00
|
|
|
spin_lock_init(&sch->busylock);
|
2012-09-04 19:02:56 -06:00
|
|
|
lockdep_set_class(&sch->busylock,
|
|
|
|
dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
|
|
|
|
|
2018-05-15 08:24:36 -06:00
|
|
|
/* seqlock has the same scope of busylock, for NOLOCK qdisc */
|
|
|
|
spin_lock_init(&sch->seqlock);
|
|
|
|
lockdep_set_class(&sch->busylock,
|
|
|
|
dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
|
|
|
|
|
2016-06-06 10:37:15 -06:00
|
|
|
seqcount_init(&sch->running);
|
|
|
|
lockdep_set_class(&sch->running,
|
|
|
|
dev->qdisc_running_key ?: &qdisc_running_key);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
sch->ops = ops;
|
2017-12-07 10:55:26 -07:00
|
|
|
sch->flags = ops->static_flags;
|
2005-04-16 16:20:36 -06:00
|
|
|
sch->enqueue = ops->enqueue;
|
|
|
|
sch->dequeue = ops->dequeue;
|
2008-07-08 17:55:56 -06:00
|
|
|
sch->dev_queue = dev_queue;
|
2019-03-22 09:01:55 -06:00
|
|
|
sch->empty = true;
|
2012-09-04 19:02:56 -06:00
|
|
|
dev_hold(dev);
|
2017-07-04 06:53:07 -06:00
|
|
|
refcount_set(&sch->refcnt, 1);
|
2005-07-05 15:15:09 -06:00
|
|
|
|
|
|
|
return sch;
|
2017-12-07 10:55:26 -07:00
|
|
|
errout1:
|
|
|
|
kfree(p);
|
2005-07-05 15:15:09 -06:00
|
|
|
errout:
|
2008-06-27 20:51:35 -06:00
|
|
|
return ERR_PTR(err);
|
2005-07-05 15:15:09 -06:00
|
|
|
}
|
|
|
|
|
2010-10-16 07:04:08 -06:00
|
|
|
struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
|
2013-08-31 11:15:50 -06:00
|
|
|
const struct Qdisc_ops *ops,
|
2017-12-20 10:35:21 -07:00
|
|
|
unsigned int parentid,
|
|
|
|
struct netlink_ext_ack *extack)
|
2005-07-05 15:15:09 -06:00
|
|
|
{
|
|
|
|
struct Qdisc *sch;
|
2007-02-09 07:25:16 -07:00
|
|
|
|
2017-12-20 10:35:21 -07:00
|
|
|
if (!try_module_get(ops->owner)) {
|
|
|
|
NL_SET_ERR_MSG(extack, "Failed to increase module reference counter");
|
2016-08-24 10:39:02 -06:00
|
|
|
return NULL;
|
2017-12-20 10:35:21 -07:00
|
|
|
}
|
2013-08-27 17:19:08 -06:00
|
|
|
|
2017-12-20 10:35:21 -07:00
|
|
|
sch = qdisc_alloc(dev_queue, ops, extack);
|
2016-08-24 10:39:02 -06:00
|
|
|
if (IS_ERR(sch)) {
|
|
|
|
module_put(ops->owner);
|
|
|
|
return NULL;
|
|
|
|
}
|
2006-11-29 18:35:18 -07:00
|
|
|
sch->parent = parentid;
|
2005-07-05 15:15:09 -06:00
|
|
|
|
2017-12-20 10:35:21 -07:00
|
|
|
if (!ops->init || ops->init(sch, NULL, extack) == 0)
|
2005-04-16 16:20:36 -06:00
|
|
|
return sch;
|
|
|
|
|
2018-09-24 10:22:50 -06:00
|
|
|
qdisc_put(sch);
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(qdisc_create_dflt);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-08-02 21:02:43 -06:00
|
|
|
/* Under qdisc_lock(qdisc) and BH! */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
void qdisc_reset(struct Qdisc *qdisc)
|
|
|
|
{
|
2007-11-14 02:44:41 -07:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
2017-12-07 10:55:45 -07:00
|
|
|
struct sk_buff *skb, *tmp;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (ops->reset)
|
|
|
|
ops->reset(qdisc);
|
2008-11-03 03:52:50 -07:00
|
|
|
|
2017-12-07 10:55:45 -07:00
|
|
|
skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
|
|
|
|
__skb_unlink(skb, &qdisc->gso_skb);
|
|
|
|
kfree_skb_list(skb);
|
2009-08-05 19:44:21 -06:00
|
|
|
}
|
2017-12-07 10:55:45 -07:00
|
|
|
|
2017-12-07 10:56:23 -07:00
|
|
|
skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
|
|
|
|
__skb_unlink(skb, &qdisc->skb_bad_txq);
|
|
|
|
kfree_skb_list(skb);
|
|
|
|
}
|
|
|
|
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 00:16:52 -06:00
|
|
|
qdisc->q.qlen = 0;
|
2017-09-20 06:45:36 -06:00
|
|
|
qdisc->qstats.backlog = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-01-22 23:10:23 -07:00
|
|
|
EXPORT_SYMBOL(qdisc_reset);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
net, sched: fix panic when updating miniq {b,q}stats
While working on fixing another bug, I ran into the following panic
on arm64 by simply attaching clsact qdisc, adding a filter and running
traffic on ingress to it:
[...]
[ 178.188591] Unable to handle kernel read from unreadable memory at virtual address 810fb501f000
[ 178.197314] Mem abort info:
[ 178.200121] ESR = 0x96000004
[ 178.203168] Exception class = DABT (current EL), IL = 32 bits
[ 178.209095] SET = 0, FnV = 0
[ 178.212157] EA = 0, S1PTW = 0
[ 178.215288] Data abort info:
[ 178.218175] ISV = 0, ISS = 0x00000004
[ 178.222019] CM = 0, WnR = 0
[ 178.224997] user pgtable: 4k pages, 48-bit VAs, pgd = 0000000023cb3f33
[ 178.231531] [0000810fb501f000] *pgd=0000000000000000
[ 178.236508] Internal error: Oops: 96000004 [#1] SMP
[...]
[ 178.311855] CPU: 73 PID: 2497 Comm: ping Tainted: G W 4.15.0-rc7+ #5
[ 178.319413] Hardware name: FOXCONN R2-1221R-A4/C2U4N_MB, BIOS G31FB18A 03/31/2017
[ 178.326887] pstate: 60400005 (nZCv daif +PAN -UAO)
[ 178.331685] pc : __netif_receive_skb_core+0x49c/0xac8
[ 178.336728] lr : __netif_receive_skb+0x28/0x78
[ 178.341161] sp : ffff00002344b750
[ 178.344465] x29: ffff00002344b750 x28: ffff810fbdfd0580
[ 178.349769] x27: 0000000000000000 x26: ffff000009378000
[...]
[ 178.418715] x1 : 0000000000000054 x0 : 0000000000000000
[ 178.424020] Process ping (pid: 2497, stack limit = 0x000000009f0a3ff4)
[ 178.430537] Call trace:
[ 178.432976] __netif_receive_skb_core+0x49c/0xac8
[ 178.437670] __netif_receive_skb+0x28/0x78
[ 178.441757] process_backlog+0x9c/0x160
[ 178.445584] net_rx_action+0x2f8/0x3f0
[...]
Reason is that sch_ingress and sch_clsact are doing mini_qdisc_pair_init()
which sets up miniq pointers to cpu_{b,q}stats from the underlying qdisc.
Problem is that this cannot work since they are actually set up right after
the qdisc ->init() callback in qdisc_create(), so first packet going into
sch_handle_ingress() tries to call mini_qdisc_bstats_cpu_update() and we
therefore panic.
In order to fix this, allocation of {b,q}stats needs to happen before we
call into ->init(). In net-next, there's already such option through commit
d59f5ffa59d8 ("net: sched: a dflt qdisc may be used with per cpu stats").
However, the bug needs to be fixed in net still for 4.15. Thus, include
these bits to reduce any merge churn and reuse the static_flags field to
set TCQ_F_CPUSTATS, and remove the allocation from qdisc_create() since
there is no other user left. Prashant Bhole ran into the same issue but
for net-next, thus adding him below as well as co-author. Same issue was
also reported by Sandipan Das when using bcc.
Fixes: 46209401f8f6 ("net: core: introduce mini_Qdisc and eliminate usage of tp->q for clsact fastpath")
Reference: https://lists.iovisor.org/pipermail/iovisor-dev/2018-January/001190.html
Reported-by: Sandipan Das <sandipan@linux.vnet.ibm.com>
Co-authored-by: Prashant Bhole <bhole_prashant_q7@lab.ntt.co.jp>
Co-authored-by: John Fastabend <john.fastabend@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Cc: Jiri Pirko <jiri@resnulli.us>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-01-15 15:12:09 -07:00
|
|
|
void qdisc_free(struct Qdisc *qdisc)
|
2010-03-31 01:06:04 -06:00
|
|
|
{
|
2016-01-05 10:11:36 -07:00
|
|
|
if (qdisc_is_percpu_stats(qdisc)) {
|
2014-09-28 12:52:56 -06:00
|
|
|
free_percpu(qdisc->cpu_bstats);
|
2016-01-05 10:11:36 -07:00
|
|
|
free_percpu(qdisc->cpu_qstats);
|
|
|
|
}
|
2014-09-28 12:52:56 -06:00
|
|
|
|
2010-03-31 01:06:04 -06:00
|
|
|
kfree((char *) qdisc - qdisc->padded);
|
|
|
|
}
|
|
|
|
|
2018-09-27 08:47:56 -06:00
|
|
|
static void qdisc_free_cb(struct rcu_head *head)
|
2018-09-24 10:22:51 -06:00
|
|
|
{
|
|
|
|
struct Qdisc *q = container_of(head, struct Qdisc, rcu);
|
|
|
|
|
|
|
|
qdisc_free(q);
|
|
|
|
}
|
|
|
|
|
2018-09-24 10:22:50 -06:00
|
|
|
static void qdisc_destroy(struct Qdisc *qdisc)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-07-17 01:47:45 -06:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
2017-12-07 10:55:45 -07:00
|
|
|
struct sk_buff *skb, *tmp;
|
2008-07-17 01:47:45 -06:00
|
|
|
|
2008-07-20 19:13:01 -06:00
|
|
|
#ifdef CONFIG_NET_SCHED
|
2016-08-10 03:05:15 -06:00
|
|
|
qdisc_hash_del(qdisc);
|
2008-08-22 04:24:05 -06:00
|
|
|
|
2011-01-19 20:48:19 -07:00
|
|
|
qdisc_put_stab(rtnl_dereference(qdisc->stab));
|
2008-07-20 19:13:01 -06:00
|
|
|
#endif
|
2016-12-04 10:48:16 -07:00
|
|
|
gen_kill_estimator(&qdisc->rate_est);
|
2008-07-17 01:47:45 -06:00
|
|
|
if (ops->reset)
|
|
|
|
ops->reset(qdisc);
|
|
|
|
if (ops->destroy)
|
|
|
|
ops->destroy(qdisc);
|
|
|
|
|
|
|
|
module_put(ops->owner);
|
|
|
|
dev_put(qdisc_dev(qdisc));
|
|
|
|
|
2017-12-07 10:55:45 -07:00
|
|
|
skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
|
|
|
|
__skb_unlink(skb, &qdisc->gso_skb);
|
|
|
|
kfree_skb_list(skb);
|
|
|
|
}
|
|
|
|
|
2017-12-07 10:56:23 -07:00
|
|
|
skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
|
|
|
|
__skb_unlink(skb, &qdisc->skb_bad_txq);
|
|
|
|
kfree_skb_list(skb);
|
|
|
|
}
|
|
|
|
|
2018-09-24 10:22:51 -06:00
|
|
|
call_rcu(&qdisc->rcu, qdisc_free_cb);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2018-09-24 10:22:50 -06:00
|
|
|
|
|
|
|
void qdisc_put(struct Qdisc *qdisc)
|
|
|
|
{
|
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN ||
|
|
|
|
!refcount_dec_and_test(&qdisc->refcnt))
|
|
|
|
return;
|
|
|
|
|
|
|
|
qdisc_destroy(qdisc);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(qdisc_put);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2018-09-24 10:22:51 -06:00
|
|
|
/* Version of qdisc_put() that is called with rtnl mutex unlocked.
|
|
|
|
* Intended to be used as optimization, this function only takes rtnl lock if
|
|
|
|
* qdisc reference counter reached zero.
|
|
|
|
*/
|
|
|
|
|
|
|
|
void qdisc_put_unlocked(struct Qdisc *qdisc)
|
|
|
|
{
|
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN ||
|
|
|
|
!refcount_dec_and_rtnl_lock(&qdisc->refcnt))
|
|
|
|
return;
|
|
|
|
|
|
|
|
qdisc_destroy(qdisc);
|
|
|
|
rtnl_unlock();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(qdisc_put_unlocked);
|
|
|
|
|
2009-09-04 00:41:20 -06:00
|
|
|
/* Attach toplevel qdisc to device queue. */
|
|
|
|
struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
|
|
|
|
struct Qdisc *qdisc)
|
|
|
|
{
|
|
|
|
struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
|
|
|
|
spinlock_t *root_lock;
|
|
|
|
|
|
|
|
root_lock = qdisc_lock(oqdisc);
|
|
|
|
spin_lock_bh(root_lock);
|
|
|
|
|
|
|
|
/* ... and graft new one */
|
|
|
|
if (qdisc == NULL)
|
|
|
|
qdisc = &noop_qdisc;
|
|
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
|
|
rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
|
|
|
|
|
|
|
|
spin_unlock_bh(root_lock);
|
|
|
|
|
|
|
|
return oqdisc;
|
|
|
|
}
|
2011-01-17 01:06:09 -07:00
|
|
|
EXPORT_SYMBOL(dev_graft_qdisc);
|
2009-09-04 00:41:20 -06:00
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void attach_one_default_qdisc(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_unused)
|
|
|
|
{
|
2015-08-27 13:21:39 -06:00
|
|
|
struct Qdisc *qdisc;
|
|
|
|
const struct Qdisc_ops *ops = default_qdisc_ops;
|
2008-07-17 01:34:19 -06:00
|
|
|
|
2015-08-27 13:21:39 -06:00
|
|
|
if (dev->priv_flags & IFF_NO_QUEUE)
|
|
|
|
ops = &noqueue_qdisc_ops;
|
|
|
|
|
2017-12-20 10:35:21 -07:00
|
|
|
qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
|
2015-08-27 13:21:39 -06:00
|
|
|
if (!qdisc) {
|
|
|
|
netdev_info(dev, "activation failed\n");
|
|
|
|
return;
|
2008-07-17 01:34:19 -06:00
|
|
|
}
|
2015-08-27 13:21:39 -06:00
|
|
|
if (!netif_is_multiqueue(dev))
|
2015-12-01 21:08:51 -07:00
|
|
|
qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
|
2008-07-17 01:34:19 -06:00
|
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
|
|
}
|
|
|
|
|
2009-09-06 02:58:51 -06:00
|
|
|
static void attach_default_qdiscs(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
|
|
|
txq = netdev_get_tx_queue(dev, 0);
|
|
|
|
|
2015-08-13 11:01:07 -06:00
|
|
|
if (!netif_is_multiqueue(dev) ||
|
|
|
|
dev->priv_flags & IFF_NO_QUEUE) {
|
2009-09-06 02:58:51 -06:00
|
|
|
netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
|
|
|
|
dev->qdisc = txq->qdisc_sleeping;
|
2017-08-24 22:12:28 -06:00
|
|
|
qdisc_refcount_inc(dev->qdisc);
|
2009-09-06 02:58:51 -06:00
|
|
|
} else {
|
2017-12-20 10:35:21 -07:00
|
|
|
qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
|
2009-09-06 02:58:51 -06:00
|
|
|
if (qdisc) {
|
|
|
|
dev->qdisc = qdisc;
|
2013-12-12 16:41:56 -07:00
|
|
|
qdisc->ops->attach(qdisc);
|
2009-09-06 02:58:51 -06:00
|
|
|
}
|
|
|
|
}
|
2016-08-10 03:05:15 -06:00
|
|
|
#ifdef CONFIG_NET_SCHED
|
2017-04-04 19:51:30 -06:00
|
|
|
if (dev->qdisc != &noop_qdisc)
|
2017-03-08 08:03:32 -07:00
|
|
|
qdisc_hash_add(dev->qdisc, false);
|
2016-08-10 03:05:15 -06:00
|
|
|
#endif
|
2009-09-06 02:58:51 -06:00
|
|
|
}
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void transition_one_qdisc(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_need_watchdog)
|
|
|
|
{
|
2008-07-17 01:53:03 -06:00
|
|
|
struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
|
2008-07-17 01:34:19 -06:00
|
|
|
int *need_watchdog_p = _need_watchdog;
|
|
|
|
|
2008-08-17 22:51:03 -06:00
|
|
|
if (!(new_qdisc->flags & TCQ_F_BUILTIN))
|
|
|
|
clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
|
|
|
|
|
2008-07-17 01:53:03 -06:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
|
2015-08-27 13:21:39 -06:00
|
|
|
if (need_watchdog_p) {
|
2009-05-17 21:55:16 -06:00
|
|
|
dev_queue->trans_start = 0;
|
2008-07-17 01:34:19 -06:00
|
|
|
*need_watchdog_p = 1;
|
2009-05-17 21:55:16 -06:00
|
|
|
}
|
2008-07-17 01:34:19 -06:00
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
void dev_activate(struct net_device *dev)
|
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
int need_watchdog;
|
2008-07-08 18:42:10 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* No queueing discipline is attached to device;
|
2013-08-27 17:19:08 -06:00
|
|
|
* create default one for devices, which need queueing
|
|
|
|
* and noqueue_qdisc for virtual interfaces
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
2009-09-06 02:58:51 -06:00
|
|
|
if (dev->qdisc == &noop_qdisc)
|
|
|
|
attach_default_qdiscs(dev);
|
2009-09-04 00:41:18 -06:00
|
|
|
|
2005-05-03 17:18:52 -06:00
|
|
|
if (!netif_carrier_ok(dev))
|
|
|
|
/* Delay activation until next carrier-on event */
|
|
|
|
return;
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
need_watchdog = 0;
|
|
|
|
netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
|
2010-10-02 00:11:55 -06:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
|
2008-07-17 01:34:19 -06:00
|
|
|
|
|
|
|
if (need_watchdog) {
|
2016-05-03 08:33:13 -06:00
|
|
|
netif_trans_update(dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
dev_watchdog_up(dev);
|
|
|
|
}
|
2008-07-08 18:42:10 -06:00
|
|
|
}
|
2011-01-17 01:06:09 -07:00
|
|
|
EXPORT_SYMBOL(dev_activate);
|
2008-07-08 18:42:10 -06:00
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void dev_deactivate_queue(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_qdisc_default)
|
2008-07-08 18:42:10 -06:00
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
2008-07-09 00:10:33 -06:00
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
2014-09-12 21:04:52 -06:00
|
|
|
qdisc = rtnl_dereference(dev_queue->qdisc);
|
2008-07-08 18:42:10 -06:00
|
|
|
if (qdisc) {
|
2018-05-15 08:24:36 -06:00
|
|
|
bool nolock = qdisc->flags & TCQ_F_NOLOCK;
|
|
|
|
|
|
|
|
if (nolock)
|
|
|
|
spin_lock_bh(&qdisc->seqlock);
|
2008-07-17 01:53:03 -06:00
|
|
|
spin_lock_bh(qdisc_lock(qdisc));
|
|
|
|
|
2008-08-17 22:51:03 -06:00
|
|
|
if (!(qdisc->flags & TCQ_F_BUILTIN))
|
|
|
|
set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
|
|
|
|
|
2008-08-27 03:22:07 -06:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
2008-07-08 18:42:10 -06:00
|
|
|
qdisc_reset(qdisc);
|
2008-07-15 21:14:35 -06:00
|
|
|
|
2008-07-17 01:53:03 -06:00
|
|
|
spin_unlock_bh(qdisc_lock(qdisc));
|
2018-05-15 08:24:36 -06:00
|
|
|
if (nolock)
|
|
|
|
spin_unlock_bh(&qdisc->seqlock);
|
2008-07-08 18:42:10 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-08-17 22:58:07 -06:00
|
|
|
static bool some_qdisc_is_busy(struct net_device *dev)
|
2008-07-17 01:34:19 -06:00
|
|
|
{
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
struct netdev_queue *dev_queue;
|
2008-07-16 02:42:40 -06:00
|
|
|
spinlock_t *root_lock;
|
2008-07-16 01:56:32 -06:00
|
|
|
struct Qdisc *q;
|
2008-07-17 01:34:19 -06:00
|
|
|
int val;
|
|
|
|
|
|
|
|
dev_queue = netdev_get_tx_queue(dev, i);
|
2008-08-13 16:18:38 -06:00
|
|
|
q = dev_queue->qdisc_sleeping;
|
2008-07-17 01:34:19 -06:00
|
|
|
|
2018-05-15 02:50:31 -06:00
|
|
|
root_lock = qdisc_lock(q);
|
|
|
|
spin_lock_bh(root_lock);
|
2008-07-17 01:34:19 -06:00
|
|
|
|
2018-05-15 02:50:31 -06:00
|
|
|
val = (qdisc_is_running(q) ||
|
|
|
|
test_bit(__QDISC_STATE_SCHED, &q->state));
|
2008-07-17 01:34:19 -06:00
|
|
|
|
2018-05-15 02:50:31 -06:00
|
|
|
spin_unlock_bh(root_lock);
|
2008-07-17 01:34:19 -06:00
|
|
|
|
|
|
|
if (val)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-12-07 10:56:04 -07:00
|
|
|
static void dev_qdisc_reset(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *none)
|
|
|
|
{
|
|
|
|
struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
|
|
|
|
|
|
|
|
if (qdisc)
|
|
|
|
qdisc_reset(qdisc);
|
|
|
|
}
|
|
|
|
|
2011-05-19 17:42:09 -06:00
|
|
|
/**
|
|
|
|
* dev_deactivate_many - deactivate transmissions on several devices
|
|
|
|
* @head: list of devices to deactivate
|
|
|
|
*
|
|
|
|
* This function returns only when all outstanding transmissions
|
|
|
|
* have completed, unless all devices are in dismantle phase.
|
|
|
|
*/
|
2010-12-13 05:44:07 -07:00
|
|
|
void dev_deactivate_many(struct list_head *head)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2010-12-13 05:44:07 -07:00
|
|
|
struct net_device *dev;
|
2007-05-10 15:12:47 -06:00
|
|
|
|
2013-10-05 20:26:05 -06:00
|
|
|
list_for_each_entry(dev, head, close_list) {
|
2010-12-13 05:44:07 -07:00
|
|
|
netdev_for_each_tx_queue(dev, dev_deactivate_queue,
|
|
|
|
&noop_qdisc);
|
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
dev_deactivate_queue(dev, dev_ingress_queue(dev),
|
|
|
|
&noop_qdisc);
|
|
|
|
|
|
|
|
dev_watchdog_down(dev);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2011-05-19 17:42:09 -06:00
|
|
|
/* Wait for outstanding qdisc-less dev_queue_xmit calls.
|
|
|
|
* This is avoided if all devices are in dismantle phase :
|
|
|
|
* Caller will call synchronize_net() for us
|
|
|
|
*/
|
2017-12-07 10:56:04 -07:00
|
|
|
synchronize_net();
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-06-22 03:28:18 -06:00
|
|
|
/* Wait for outstanding qdisc_run calls. */
|
2017-12-07 10:56:04 -07:00
|
|
|
list_for_each_entry(dev, head, close_list) {
|
2010-12-13 05:44:07 -07:00
|
|
|
while (some_qdisc_is_busy(dev))
|
|
|
|
yield();
|
2017-12-07 10:56:04 -07:00
|
|
|
/* The new qdisc is assigned at this point so we can safely
|
|
|
|
* unwind stale skb lists and qdisc statistics
|
|
|
|
*/
|
|
|
|
netdev_for_each_tx_queue(dev, dev_qdisc_reset, NULL);
|
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
dev_qdisc_reset(dev, dev_ingress_queue(dev), NULL);
|
|
|
|
}
|
2010-12-13 05:44:07 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void dev_deactivate(struct net_device *dev)
|
|
|
|
{
|
|
|
|
LIST_HEAD(single);
|
|
|
|
|
2013-10-05 20:26:05 -06:00
|
|
|
list_add(&dev->close_list, &single);
|
2010-12-13 05:44:07 -07:00
|
|
|
dev_deactivate_many(&single);
|
2011-02-20 12:49:45 -07:00
|
|
|
list_del(&single);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2011-01-17 01:06:09 -07:00
|
|
|
EXPORT_SYMBOL(dev_deactivate);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2018-01-25 19:26:23 -07:00
|
|
|
static int qdisc_change_tx_queue_len(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue)
|
|
|
|
{
|
|
|
|
struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
|
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
|
|
|
|
|
|
|
if (ops->change_tx_queue_len)
|
|
|
|
return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int dev_qdisc_change_tx_queue_len(struct net_device *dev)
|
|
|
|
{
|
|
|
|
bool up = dev->flags & IFF_UP;
|
|
|
|
unsigned int i;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
if (up)
|
|
|
|
dev_deactivate(dev);
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]);
|
|
|
|
|
|
|
|
/* TODO: revert changes on a partial failure */
|
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (up)
|
|
|
|
dev_activate(dev);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2008-07-08 18:42:10 -06:00
|
|
|
static void dev_init_scheduler_queue(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
2008-07-17 01:34:19 -06:00
|
|
|
void *_qdisc)
|
2008-07-08 18:42:10 -06:00
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
struct Qdisc *qdisc = _qdisc;
|
|
|
|
|
2014-09-12 21:04:52 -06:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc);
|
2008-07-08 18:42:10 -06:00
|
|
|
dev_queue->qdisc_sleeping = qdisc;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
void dev_init_scheduler(struct net_device *dev)
|
|
|
|
{
|
2009-09-04 00:41:18 -06:00
|
|
|
dev->qdisc = &noop_qdisc;
|
2008-07-17 01:34:19 -06:00
|
|
|
netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
|
2010-10-02 00:11:55 -06:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2017-10-16 18:29:17 -06:00
|
|
|
timer_setup(&dev->watchdog_timer, dev_watchdog, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-17 01:34:19 -06:00
|
|
|
static void shutdown_scheduler_queue(struct net_device *dev,
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
void *_qdisc_default)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-07-08 18:42:10 -06:00
|
|
|
struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
|
2008-07-17 01:34:19 -06:00
|
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
2008-07-08 18:42:10 -06:00
|
|
|
|
|
|
|
if (qdisc) {
|
2008-08-27 03:22:07 -06:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
2008-07-08 18:42:10 -06:00
|
|
|
dev_queue->qdisc_sleeping = qdisc_default;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2018-09-24 10:22:50 -06:00
|
|
|
qdisc_put(qdisc);
|
2007-02-09 07:25:16 -07:00
|
|
|
}
|
2008-07-08 18:42:10 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void dev_shutdown(struct net_device *dev)
|
|
|
|
{
|
2008-07-17 01:34:19 -06:00
|
|
|
netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
|
2010-10-02 00:11:55 -06:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
|
2018-09-24 10:22:50 -06:00
|
|
|
qdisc_put(dev->qdisc);
|
2009-09-04 00:41:18 -06:00
|
|
|
dev->qdisc = &noop_qdisc;
|
|
|
|
|
2008-07-25 22:43:18 -06:00
|
|
|
WARN_ON(timer_pending(&dev->watchdog_timer));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2013-02-11 17:12:03 -07:00
|
|
|
|
2013-06-02 07:55:05 -06:00
|
|
|
void psched_ratecfg_precompute(struct psched_ratecfg *r,
|
2013-09-19 10:10:03 -06:00
|
|
|
const struct tc_ratespec *conf,
|
|
|
|
u64 rate64)
|
2013-02-11 17:12:03 -07:00
|
|
|
{
|
2013-06-02 07:55:05 -06:00
|
|
|
memset(r, 0, sizeof(*r));
|
|
|
|
r->overhead = conf->overhead;
|
2013-09-19 10:10:03 -06:00
|
|
|
r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
|
2013-08-14 15:47:11 -06:00
|
|
|
r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
|
2013-02-11 17:12:03 -07:00
|
|
|
r->mult = 1;
|
|
|
|
/*
|
2013-06-06 14:56:19 -06:00
|
|
|
* The deal here is to replace a divide by a reciprocal one
|
|
|
|
* in fast path (a reciprocal divide is a multiply and a shift)
|
|
|
|
*
|
|
|
|
* Normal formula would be :
|
|
|
|
* time_in_ns = (NSEC_PER_SEC * len) / rate_bps
|
|
|
|
*
|
|
|
|
* We compute mult/shift to use instead :
|
|
|
|
* time_in_ns = (len * mult) >> shift;
|
|
|
|
*
|
|
|
|
* We try to get the highest possible mult value for accuracy,
|
|
|
|
* but have to make sure no overflows will ever happen.
|
2013-02-11 17:12:03 -07:00
|
|
|
*/
|
2013-06-06 14:56:19 -06:00
|
|
|
if (r->rate_bytes_ps > 0) {
|
|
|
|
u64 factor = NSEC_PER_SEC;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
r->mult = div64_u64(factor, r->rate_bytes_ps);
|
|
|
|
if (r->mult & (1U << 31) || factor & (1ULL << 63))
|
2013-02-11 17:12:03 -07:00
|
|
|
break;
|
2013-06-06 14:56:19 -06:00
|
|
|
factor <<= 1;
|
|
|
|
r->shift++;
|
2013-02-11 17:12:03 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(psched_ratecfg_precompute);
|
2017-11-03 04:46:25 -06:00
|
|
|
|
|
|
|
static void mini_qdisc_rcu_func(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
|
|
|
|
struct tcf_proto *tp_head)
|
|
|
|
{
|
2019-02-11 01:55:38 -07:00
|
|
|
/* Protected with chain0->filter_chain_lock.
|
|
|
|
* Can't access chain directly because tp_head can be NULL.
|
|
|
|
*/
|
|
|
|
struct mini_Qdisc *miniq_old =
|
|
|
|
rcu_dereference_protected(*miniqp->p_miniq, 1);
|
2017-11-03 04:46:25 -06:00
|
|
|
struct mini_Qdisc *miniq;
|
|
|
|
|
|
|
|
if (!tp_head) {
|
|
|
|
RCU_INIT_POINTER(*miniqp->p_miniq, NULL);
|
2017-12-21 00:26:24 -07:00
|
|
|
/* Wait for flying RCU callback before it is freed. */
|
2018-11-06 20:40:39 -07:00
|
|
|
rcu_barrier();
|
2017-11-03 04:46:25 -06:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
miniq = !miniq_old || miniq_old == &miniqp->miniq2 ?
|
|
|
|
&miniqp->miniq1 : &miniqp->miniq2;
|
|
|
|
|
|
|
|
/* We need to make sure that readers won't see the miniq
|
2018-11-06 20:40:39 -07:00
|
|
|
* we are about to modify. So wait until previous call_rcu callback
|
2017-11-03 04:46:25 -06:00
|
|
|
* is done.
|
|
|
|
*/
|
2018-11-06 20:40:39 -07:00
|
|
|
rcu_barrier();
|
2017-11-03 04:46:25 -06:00
|
|
|
miniq->filter_list = tp_head;
|
|
|
|
rcu_assign_pointer(*miniqp->p_miniq, miniq);
|
|
|
|
|
|
|
|
if (miniq_old)
|
2017-12-21 00:26:24 -07:00
|
|
|
/* This is counterpart of the rcu barriers above. We need to
|
2017-11-03 04:46:25 -06:00
|
|
|
* block potential new user of miniq_old until all readers
|
|
|
|
* are not seeing it.
|
|
|
|
*/
|
2018-11-06 20:40:39 -07:00
|
|
|
call_rcu(&miniq_old->rcu, mini_qdisc_rcu_func);
|
2017-11-03 04:46:25 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mini_qdisc_pair_swap);
|
|
|
|
|
|
|
|
void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
|
|
|
|
struct mini_Qdisc __rcu **p_miniq)
|
|
|
|
{
|
|
|
|
miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
|
|
|
|
miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
|
|
|
|
miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
|
|
|
|
miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;
|
|
|
|
miniqp->p_miniq = p_miniq;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mini_qdisc_pair_init);
|