2018-05-02 05:01:23 -06:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/* XDP sockets
|
|
|
|
*
|
|
|
|
* AF_XDP sockets allows a channel between XDP programs and userspace
|
|
|
|
* applications.
|
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
*
|
|
|
|
* Author(s): Björn Töpel <bjorn.topel@intel.com>
|
|
|
|
* Magnus Karlsson <magnus.karlsson@intel.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
|
|
|
|
|
|
|
|
#include <linux/if_xdp.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/sched/mm.h>
|
|
|
|
#include <linux/sched/signal.h>
|
|
|
|
#include <linux/sched/task.h>
|
|
|
|
#include <linux/socket.h>
|
|
|
|
#include <linux/file.h>
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <linux/net.h>
|
|
|
|
#include <linux/netdevice.h>
|
2018-06-04 06:05:57 -06:00
|
|
|
#include <linux/rculist.h>
|
2020-05-20 13:20:51 -06:00
|
|
|
#include <net/xdp_sock_drv.h>
|
2018-05-02 05:01:25 -06:00
|
|
|
#include <net/xdp.h>
|
2018-05-02 05:01:23 -06:00
|
|
|
|
2018-05-02 05:01:24 -06:00
|
|
|
#include "xsk_queue.h"
|
2018-05-02 05:01:23 -06:00
|
|
|
#include "xdp_umem.h"
|
2019-01-24 11:59:39 -07:00
|
|
|
#include "xsk.h"
|
2018-05-02 05:01:23 -06:00
|
|
|
|
2018-05-02 05:01:34 -06:00
|
|
|
#define TX_BATCH_SIZE 16
|
|
|
|
|
2019-12-18 23:10:02 -07:00
|
|
|
static DEFINE_PER_CPU(struct list_head, xskmap_flush_list);
|
|
|
|
|
2018-05-02 05:01:28 -06:00
|
|
|
bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
|
|
|
|
{
|
2018-06-04 06:05:55 -06:00
|
|
|
return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) &&
|
|
|
|
READ_ONCE(xs->umem->fq);
|
2018-05-02 05:01:28 -06:00
|
|
|
}
|
|
|
|
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
void xsk_set_rx_need_wakeup(struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
if (umem->need_wakeup & XDP_WAKEUP_RX)
|
|
|
|
return;
|
|
|
|
|
|
|
|
umem->fq->ring->flags |= XDP_RING_NEED_WAKEUP;
|
|
|
|
umem->need_wakeup |= XDP_WAKEUP_RX;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_set_rx_need_wakeup);
|
|
|
|
|
|
|
|
void xsk_set_tx_need_wakeup(struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
if (umem->need_wakeup & XDP_WAKEUP_TX)
|
|
|
|
return;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2020-05-04 07:33:51 -06:00
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_tx_list, list) {
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
umem->need_wakeup |= XDP_WAKEUP_TX;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_set_tx_need_wakeup);
|
|
|
|
|
|
|
|
void xsk_clear_rx_need_wakeup(struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
if (!(umem->need_wakeup & XDP_WAKEUP_RX))
|
|
|
|
return;
|
|
|
|
|
|
|
|
umem->fq->ring->flags &= ~XDP_RING_NEED_WAKEUP;
|
|
|
|
umem->need_wakeup &= ~XDP_WAKEUP_RX;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_clear_rx_need_wakeup);
|
|
|
|
|
|
|
|
void xsk_clear_tx_need_wakeup(struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
if (!(umem->need_wakeup & XDP_WAKEUP_TX))
|
|
|
|
return;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2020-05-04 07:33:51 -06:00
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_tx_list, list) {
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
xs->tx->ring->flags &= ~XDP_RING_NEED_WAKEUP;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
umem->need_wakeup &= ~XDP_WAKEUP_TX;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_clear_tx_need_wakeup);
|
|
|
|
|
|
|
|
bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
return umem->flags & XDP_UMEM_USES_NEED_WAKEUP;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_umem_uses_need_wakeup);
|
|
|
|
|
2020-05-20 13:21:02 -06:00
|
|
|
void xp_release(struct xdp_buff_xsk *xskb)
|
|
|
|
{
|
|
|
|
xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 xp_get_handle(struct xdp_buff_xsk *xskb)
|
|
|
|
{
|
|
|
|
u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start;
|
|
|
|
|
|
|
|
offset += xskb->pool->headroom;
|
|
|
|
if (!xskb->pool->unaligned)
|
|
|
|
return xskb->orig_addr + offset;
|
|
|
|
return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
|
|
|
|
}
|
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
{
|
2020-05-20 13:20:53 -06:00
|
|
|
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
|
|
|
|
u64 addr;
|
|
|
|
int err;
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
addr = xp_get_handle(xskb);
|
|
|
|
err = xskq_prod_reserve_desc(xs->rx, addr, len);
|
|
|
|
if (err) {
|
|
|
|
xs->rx_dropped++;
|
|
|
|
return err;
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
}
|
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
xp_release(xskb);
|
|
|
|
return 0;
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
}
|
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
static void xsk_copy_xdp(struct xdp_buff *to, struct xdp_buff *from, u32 len)
|
2018-05-02 05:01:27 -06:00
|
|
|
{
|
2020-05-20 13:20:53 -06:00
|
|
|
void *from_buf, *to_buf;
|
2018-08-30 07:12:48 -06:00
|
|
|
u32 metalen;
|
2018-05-02 05:01:27 -06:00
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
if (unlikely(xdp_data_meta_unsupported(from))) {
|
|
|
|
from_buf = from->data;
|
|
|
|
to_buf = to->data;
|
2018-08-30 07:12:48 -06:00
|
|
|
metalen = 0;
|
|
|
|
} else {
|
2020-05-20 13:20:53 -06:00
|
|
|
from_buf = from->data_meta;
|
|
|
|
metalen = from->data - from->data_meta;
|
|
|
|
to_buf = to->data - metalen;
|
2018-08-30 07:12:48 -06:00
|
|
|
}
|
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
memcpy(to_buf, from_buf, len + metalen);
|
2018-05-02 05:01:27 -06:00
|
|
|
}
|
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len,
|
|
|
|
bool explicit_free)
|
2018-05-02 05:01:27 -06:00
|
|
|
{
|
2020-05-20 13:20:53 -06:00
|
|
|
struct xdp_buff *xsk_xdp;
|
|
|
|
int err;
|
2018-05-02 05:01:27 -06:00
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
if (len > xsk_umem_get_rx_frame_size(xs->umem)) {
|
|
|
|
xs->rx_dropped++;
|
|
|
|
return -ENOSPC;
|
|
|
|
}
|
|
|
|
|
|
|
|
xsk_xdp = xsk_buff_alloc(xs->umem);
|
|
|
|
if (!xsk_xdp) {
|
2018-06-04 06:05:55 -06:00
|
|
|
xs->rx_dropped++;
|
2020-05-20 13:20:53 -06:00
|
|
|
return -ENOSPC;
|
|
|
|
}
|
2018-05-02 05:01:27 -06:00
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
xsk_copy_xdp(xsk_xdp, xdp, len);
|
|
|
|
err = __xsk_rcv_zc(xs, xsk_xdp, len);
|
|
|
|
if (err) {
|
|
|
|
xsk_buff_free(xsk_xdp);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
if (explicit_free)
|
|
|
|
xdp_return_buff(xdp);
|
|
|
|
return 0;
|
2018-05-02 05:01:27 -06:00
|
|
|
}
|
|
|
|
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
static bool xsk_is_bound(struct xdp_sock *xs)
|
|
|
|
{
|
|
|
|
if (READ_ONCE(xs->state) == XSK_BOUND) {
|
|
|
|
/* Matches smp_wmb() in bind(). */
|
|
|
|
smp_rmb();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,
|
|
|
|
bool explicit_free)
|
2018-06-04 06:05:55 -06:00
|
|
|
{
|
|
|
|
u32 len;
|
|
|
|
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
if (!xsk_is_bound(xs))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2018-06-04 06:05:55 -06:00
|
|
|
if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
len = xdp->data_end - xdp->data;
|
|
|
|
|
2020-05-20 13:21:00 -06:00
|
|
|
return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?
|
2020-05-20 13:20:53 -06:00
|
|
|
__xsk_rcv_zc(xs, xdp, len) :
|
|
|
|
__xsk_rcv(xs, xdp, len, explicit_free);
|
2018-06-04 06:05:55 -06:00
|
|
|
}
|
|
|
|
|
2019-11-01 05:03:46 -06:00
|
|
|
static void xsk_flush(struct xdp_sock *xs)
|
2018-05-02 05:01:27 -06:00
|
|
|
{
|
2019-12-19 05:39:23 -07:00
|
|
|
xskq_prod_submit(xs->rx);
|
2020-02-10 08:27:12 -07:00
|
|
|
__xskq_cons_release(xs->umem->fq);
|
2020-01-20 02:29:17 -07:00
|
|
|
sock_def_readable(&xs->sk);
|
2018-05-02 05:01:27 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
2019-07-03 06:09:16 -06:00
|
|
|
spin_lock_bh(&xs->rx_lock);
|
2020-05-20 13:20:53 -06:00
|
|
|
err = xsk_rcv(xs, xdp, false);
|
|
|
|
xsk_flush(xs);
|
2019-07-03 06:09:16 -06:00
|
|
|
spin_unlock_bh(&xs->rx_lock);
|
2018-05-02 05:01:27 -06:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2019-12-18 23:10:02 -07:00
|
|
|
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
|
2019-11-01 05:03:46 -06:00
|
|
|
{
|
2019-12-18 23:10:02 -07:00
|
|
|
struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
|
2019-11-01 05:03:46 -06:00
|
|
|
int err;
|
|
|
|
|
2020-05-20 13:20:53 -06:00
|
|
|
err = xsk_rcv(xs, xdp, true);
|
2019-11-01 05:03:46 -06:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
if (!xs->flush_node.prev)
|
|
|
|
list_add(&xs->flush_node, flush_list);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-12-18 23:10:02 -07:00
|
|
|
void __xsk_map_flush(void)
|
2019-11-01 05:03:46 -06:00
|
|
|
{
|
2019-12-18 23:10:02 -07:00
|
|
|
struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
|
2019-11-01 05:03:46 -06:00
|
|
|
struct xdp_sock *xs, *tmp;
|
|
|
|
|
|
|
|
list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
|
|
|
|
xsk_flush(xs);
|
|
|
|
__list_del_clearprev(&xs->flush_node);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-04 06:05:57 -06:00
|
|
|
void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
|
|
|
|
{
|
2019-12-19 05:39:23 -07:00
|
|
|
xskq_prod_submit_n(umem->cq, nb_entries);
|
2018-06-04 06:05:57 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_umem_complete_tx);
|
|
|
|
|
|
|
|
void xsk_umem_consume_tx_done(struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2020-05-04 07:33:51 -06:00
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_tx_list, list) {
|
2020-02-10 08:27:12 -07:00
|
|
|
__xskq_cons_release(xs->tx);
|
2018-06-04 06:05:57 -06:00
|
|
|
xs->sk.sk_write_space(&xs->sk);
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_umem_consume_tx_done);
|
|
|
|
|
2019-06-26 08:35:28 -06:00
|
|
|
bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc)
|
2018-06-04 06:05:57 -06:00
|
|
|
{
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2020-05-04 07:33:51 -06:00
|
|
|
list_for_each_entry_rcu(xs, &umem->xsk_tx_list, list) {
|
2019-12-19 05:39:26 -07:00
|
|
|
if (!xskq_cons_peek_desc(xs->tx, desc, umem))
|
2018-06-04 06:05:57 -06:00
|
|
|
continue;
|
|
|
|
|
2020-04-21 17:29:27 -06:00
|
|
|
/* This is the backpressure mechanism for the Tx path.
|
2019-12-19 05:39:30 -07:00
|
|
|
* Reserve space in the completion queue and only proceed
|
|
|
|
* if there is space in it. This avoids having to implement
|
|
|
|
* any buffering in the Tx path.
|
|
|
|
*/
|
2019-12-19 05:39:23 -07:00
|
|
|
if (xskq_prod_reserve_addr(umem->cq, desc->addr))
|
2018-06-04 06:05:57 -06:00
|
|
|
goto out;
|
|
|
|
|
2019-12-19 05:39:26 -07:00
|
|
|
xskq_cons_release(xs->tx);
|
2018-06-04 06:05:57 -06:00
|
|
|
rcu_read_unlock();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
rcu_read_unlock();
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(xsk_umem_consume_tx);
|
|
|
|
|
2019-12-17 09:20:42 -07:00
|
|
|
static int xsk_wakeup(struct xdp_sock *xs, u8 flags)
|
2018-06-04 06:05:57 -06:00
|
|
|
{
|
|
|
|
struct net_device *dev = xs->dev;
|
2019-12-17 09:20:42 -07:00
|
|
|
int err;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
err = dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
|
|
|
|
rcu_read_unlock();
|
2018-06-04 06:05:57 -06:00
|
|
|
|
2019-12-17 09:20:42 -07:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int xsk_zc_xmit(struct xdp_sock *xs)
|
|
|
|
{
|
|
|
|
return xsk_wakeup(xs, XDP_WAKEUP_TX);
|
2018-06-04 06:05:57 -06:00
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:34 -06:00
|
|
|
static void xsk_destruct_skb(struct sk_buff *skb)
|
|
|
|
{
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 05:57:13 -06:00
|
|
|
u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg;
|
2018-05-02 05:01:34 -06:00
|
|
|
struct xdp_sock *xs = xdp_sk(skb->sk);
|
2018-06-29 01:48:20 -06:00
|
|
|
unsigned long flags;
|
2018-05-02 05:01:34 -06:00
|
|
|
|
2018-06-29 01:48:20 -06:00
|
|
|
spin_lock_irqsave(&xs->tx_completion_lock, flags);
|
2019-12-19 05:39:23 -07:00
|
|
|
xskq_prod_submit_addr(xs->umem->cq, addr);
|
2018-06-29 01:48:20 -06:00
|
|
|
spin_unlock_irqrestore(&xs->tx_completion_lock, flags);
|
2018-05-02 05:01:34 -06:00
|
|
|
|
|
|
|
sock_wfree(skb);
|
|
|
|
}
|
|
|
|
|
2019-10-02 00:31:59 -06:00
|
|
|
static int xsk_generic_xmit(struct sock *sk)
|
2018-05-02 05:01:34 -06:00
|
|
|
{
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
2019-10-02 00:31:59 -06:00
|
|
|
u32 max_batch = TX_BATCH_SIZE;
|
2018-05-02 05:01:34 -06:00
|
|
|
bool sent_frame = false;
|
|
|
|
struct xdp_desc desc;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
|
2019-07-04 08:25:03 -06:00
|
|
|
if (xs->queue_id >= xs->dev->real_num_tx_queues)
|
|
|
|
goto out;
|
|
|
|
|
2019-12-19 05:39:26 -07:00
|
|
|
while (xskq_cons_peek_desc(xs->tx, &desc, xs->umem)) {
|
2018-05-02 05:01:34 -06:00
|
|
|
char *buffer;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 05:57:13 -06:00
|
|
|
u64 addr;
|
|
|
|
u32 len;
|
2018-05-02 05:01:34 -06:00
|
|
|
|
|
|
|
if (max_batch-- == 0) {
|
|
|
|
err = -EAGAIN;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2018-07-11 02:12:52 -06:00
|
|
|
len = desc.len;
|
2018-06-04 06:05:57 -06:00
|
|
|
skb = sock_alloc_send_skb(sk, len, 1, &err);
|
2018-05-02 05:01:34 -06:00
|
|
|
if (unlikely(!skb)) {
|
|
|
|
err = -EAGAIN;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
skb_put(skb, len);
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 05:57:13 -06:00
|
|
|
addr = desc.addr;
|
2020-05-20 13:20:53 -06:00
|
|
|
buffer = xsk_buff_raw_get_data(xs->umem, addr);
|
2018-05-02 05:01:34 -06:00
|
|
|
err = skb_store_bits(skb, 0, buffer, len);
|
2020-04-21 17:29:27 -06:00
|
|
|
/* This is the backpressure mechanism for the Tx path.
|
2019-12-19 05:39:30 -07:00
|
|
|
* Reserve space in the completion queue and only proceed
|
|
|
|
* if there is space in it. This avoids having to implement
|
|
|
|
* any buffering in the Tx path.
|
|
|
|
*/
|
2019-12-19 05:39:23 -07:00
|
|
|
if (unlikely(err) || xskq_prod_reserve(xs->umem->cq)) {
|
2018-05-02 05:01:34 -06:00
|
|
|
kfree_skb(skb);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
skb->dev = xs->dev;
|
|
|
|
skb->priority = sk->sk_priority;
|
|
|
|
skb->mark = sk->sk_mark;
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
skb_shinfo(skb)->destructor_arg = (void *)(long)desc.addr;
|
2018-05-02 05:01:34 -06:00
|
|
|
skb->destructor = xsk_destruct_skb;
|
|
|
|
|
|
|
|
err = dev_direct_xmit(skb, xs->queue_id);
|
2019-12-19 05:39:26 -07:00
|
|
|
xskq_cons_release(xs->tx);
|
2018-05-02 05:01:34 -06:00
|
|
|
/* Ignore NET_XMIT_CN as packet might have been sent */
|
|
|
|
if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) {
|
2018-06-29 01:48:18 -06:00
|
|
|
/* SKB completed but not sent */
|
|
|
|
err = -EBUSY;
|
2018-05-02 05:01:34 -06:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
sent_frame = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
if (sent_frame)
|
|
|
|
sk->sk_write_space(sk);
|
|
|
|
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2019-10-02 00:31:59 -06:00
|
|
|
static int __xsk_sendmsg(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
if (unlikely(!(xs->dev->flags & IFF_UP)))
|
|
|
|
return -ENETDOWN;
|
|
|
|
if (unlikely(!xs->tx))
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
return xs->zc ? xsk_zc_xmit(xs) : xsk_generic_xmit(sk);
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:34 -06:00
|
|
|
static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
|
|
|
|
{
|
2018-06-04 06:05:57 -06:00
|
|
|
bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
|
2018-05-02 05:01:34 -06:00
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
if (unlikely(!xsk_is_bound(xs)))
|
2018-05-02 05:01:34 -06:00
|
|
|
return -ENXIO;
|
2019-10-02 00:31:59 -06:00
|
|
|
if (unlikely(need_wait))
|
2018-06-04 06:05:57 -06:00
|
|
|
return -EOPNOTSUPP;
|
2018-05-02 05:01:34 -06:00
|
|
|
|
2019-10-02 00:31:59 -06:00
|
|
|
return __xsk_sendmsg(sk);
|
2018-05-02 05:01:34 -06:00
|
|
|
}
|
|
|
|
|
2019-11-19 17:10:42 -07:00
|
|
|
static __poll_t xsk_poll(struct file *file, struct socket *sock,
|
2018-06-28 10:43:44 -06:00
|
|
|
struct poll_table_struct *wait)
|
2018-05-02 05:01:27 -06:00
|
|
|
{
|
2019-11-19 17:10:42 -07:00
|
|
|
__poll_t mask = datagram_poll(file, sock, wait);
|
2019-10-02 00:31:59 -06:00
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
struct xdp_umem *umem;
|
|
|
|
|
|
|
|
if (unlikely(!xsk_is_bound(xs)))
|
|
|
|
return mask;
|
|
|
|
|
|
|
|
umem = xs->umem;
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
|
2019-10-02 00:31:59 -06:00
|
|
|
if (umem->need_wakeup) {
|
2019-12-17 09:20:42 -07:00
|
|
|
if (xs->zc)
|
|
|
|
xsk_wakeup(xs, umem->need_wakeup);
|
2019-10-02 00:31:59 -06:00
|
|
|
else
|
|
|
|
/* Poll needs to drive Tx also in copy mode */
|
|
|
|
__xsk_sendmsg(sk);
|
|
|
|
}
|
2018-05-02 05:01:27 -06:00
|
|
|
|
2019-12-19 05:39:23 -07:00
|
|
|
if (xs->rx && !xskq_prod_is_empty(xs->rx))
|
2019-11-19 17:10:42 -07:00
|
|
|
mask |= EPOLLIN | EPOLLRDNORM;
|
2019-12-19 05:39:26 -07:00
|
|
|
if (xs->tx && !xskq_cons_is_full(xs->tx))
|
2019-11-19 17:10:42 -07:00
|
|
|
mask |= EPOLLOUT | EPOLLWRNORM;
|
2018-05-02 05:01:27 -06:00
|
|
|
|
|
|
|
return mask;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:25 -06:00
|
|
|
static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
|
|
|
|
bool umem_queue)
|
2018-05-02 05:01:24 -06:00
|
|
|
{
|
|
|
|
struct xsk_queue *q;
|
|
|
|
|
|
|
|
if (entries == 0 || *queue || !is_power_of_2(entries))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2018-05-02 05:01:25 -06:00
|
|
|
q = xskq_create(entries, umem_queue);
|
2018-05-02 05:01:24 -06:00
|
|
|
if (!q)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2018-05-22 01:35:01 -06:00
|
|
|
/* Make sure queue is ready before it can be seen by others */
|
|
|
|
smp_wmb();
|
2019-09-04 05:49:10 -06:00
|
|
|
WRITE_ONCE(*queue, q);
|
2018-05-02 05:01:24 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-06-28 02:04:07 -06:00
|
|
|
static void xsk_unbind_dev(struct xdp_sock *xs)
|
|
|
|
{
|
|
|
|
struct net_device *dev = xs->dev;
|
|
|
|
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
if (xs->state != XSK_BOUND)
|
2019-06-28 02:04:07 -06:00
|
|
|
return;
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
WRITE_ONCE(xs->state, XSK_UNBOUND);
|
2019-06-28 02:04:07 -06:00
|
|
|
|
|
|
|
/* Wait for driver to stop using the xdp socket. */
|
|
|
|
xdp_del_sk_umem(xs->umem, xs);
|
|
|
|
xs->dev = NULL;
|
|
|
|
synchronize_net();
|
|
|
|
dev_put(dev);
|
|
|
|
}
|
|
|
|
|
2019-08-15 03:30:13 -06:00
|
|
|
static struct xsk_map *xsk_get_map_list_entry(struct xdp_sock *xs,
|
|
|
|
struct xdp_sock ***map_entry)
|
|
|
|
{
|
|
|
|
struct xsk_map *map = NULL;
|
|
|
|
struct xsk_map_node *node;
|
|
|
|
|
|
|
|
*map_entry = NULL;
|
|
|
|
|
|
|
|
spin_lock_bh(&xs->map_list_lock);
|
|
|
|
node = list_first_entry_or_null(&xs->map_list, struct xsk_map_node,
|
|
|
|
node);
|
|
|
|
if (node) {
|
|
|
|
WARN_ON(xsk_map_inc(node->map));
|
|
|
|
map = node->map;
|
|
|
|
*map_entry = node->map_entry;
|
|
|
|
}
|
|
|
|
spin_unlock_bh(&xs->map_list_lock);
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void xsk_delete_from_maps(struct xdp_sock *xs)
|
|
|
|
{
|
|
|
|
/* This function removes the current XDP socket from all the
|
|
|
|
* maps it resides in. We need to take extra care here, due to
|
|
|
|
* the two locks involved. Each map has a lock synchronizing
|
|
|
|
* updates to the entries, and each socket has a lock that
|
|
|
|
* synchronizes access to the list of maps (map_list). For
|
|
|
|
* deadlock avoidance the locks need to be taken in the order
|
|
|
|
* "map lock"->"socket map list lock". We start off by
|
|
|
|
* accessing the socket map list, and take a reference to the
|
|
|
|
* map to guarantee existence between the
|
|
|
|
* xsk_get_map_list_entry() and xsk_map_try_sock_delete()
|
|
|
|
* calls. Then we ask the map to remove the socket, which
|
|
|
|
* tries to remove the socket from the map. Note that there
|
|
|
|
* might be updates to the map between
|
|
|
|
* xsk_get_map_list_entry() and xsk_map_try_sock_delete().
|
|
|
|
*/
|
|
|
|
struct xdp_sock **map_entry = NULL;
|
|
|
|
struct xsk_map *map;
|
|
|
|
|
|
|
|
while ((map = xsk_get_map_list_entry(xs, &map_entry))) {
|
|
|
|
xsk_map_try_sock_delete(map, xs, map_entry);
|
|
|
|
xsk_map_put(map);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
static int xsk_release(struct socket *sock)
|
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
2018-05-02 05:01:26 -06:00
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
2018-05-02 05:01:23 -06:00
|
|
|
struct net *net;
|
|
|
|
|
|
|
|
if (!sk)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
net = sock_net(sk);
|
|
|
|
|
2019-01-24 11:59:37 -07:00
|
|
|
mutex_lock(&net->xdp.lock);
|
|
|
|
sk_del_node_init_rcu(sk);
|
|
|
|
mutex_unlock(&net->xdp.lock);
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
local_bh_disable();
|
|
|
|
sock_prot_inuse_add(net, sk->sk_prot, -1);
|
|
|
|
local_bh_enable();
|
|
|
|
|
2019-08-15 03:30:13 -06:00
|
|
|
xsk_delete_from_maps(xs);
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
mutex_lock(&xs->mutex);
|
2019-06-28 02:04:07 -06:00
|
|
|
xsk_unbind_dev(xs);
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
mutex_unlock(&xs->mutex);
|
2018-05-02 05:01:26 -06:00
|
|
|
|
2018-10-05 05:25:15 -06:00
|
|
|
xskq_destroy(xs->rx);
|
|
|
|
xskq_destroy(xs->tx);
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
sock_orphan(sk);
|
|
|
|
sock->sk = NULL;
|
|
|
|
|
|
|
|
sk_refcnt_debug_release(sk);
|
|
|
|
sock_put(sk);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:26 -06:00
|
|
|
static struct socket *xsk_lookup_xsk_from_fd(int fd)
|
|
|
|
{
|
|
|
|
struct socket *sock;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
sock = sockfd_lookup(fd, &err);
|
|
|
|
if (!sock)
|
|
|
|
return ERR_PTR(-ENOTSOCK);
|
|
|
|
|
|
|
|
if (sock->sk->sk_family != PF_XDP) {
|
|
|
|
sockfd_put(sock);
|
|
|
|
return ERR_PTR(-ENOPROTOOPT);
|
|
|
|
}
|
|
|
|
|
|
|
|
return sock;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
|
|
|
|
{
|
|
|
|
struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
2018-05-22 01:34:56 -06:00
|
|
|
struct net_device *dev;
|
2018-06-04 06:05:55 -06:00
|
|
|
u32 flags, qid;
|
2018-05-02 05:01:26 -06:00
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
if (addr_len < sizeof(struct sockaddr_xdp))
|
|
|
|
return -EINVAL;
|
|
|
|
if (sxdp->sxdp_family != AF_XDP)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2019-03-08 00:57:26 -07:00
|
|
|
flags = sxdp->sxdp_flags;
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
if (flags & ~(XDP_SHARED_UMEM | XDP_COPY | XDP_ZEROCOPY |
|
|
|
|
XDP_USE_NEED_WAKEUP))
|
2019-03-08 00:57:26 -07:00
|
|
|
return -EINVAL;
|
|
|
|
|
2019-07-08 05:03:44 -06:00
|
|
|
rtnl_lock();
|
2018-05-02 05:01:26 -06:00
|
|
|
mutex_lock(&xs->mutex);
|
2019-06-28 02:04:07 -06:00
|
|
|
if (xs->state != XSK_READY) {
|
2018-05-22 01:34:56 -06:00
|
|
|
err = -EBUSY;
|
|
|
|
goto out_release;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:26 -06:00
|
|
|
dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
|
|
|
|
if (!dev) {
|
|
|
|
err = -ENODEV;
|
|
|
|
goto out_release;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:32 -06:00
|
|
|
if (!xs->rx && !xs->tx) {
|
2018-05-02 05:01:26 -06:00
|
|
|
err = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2018-06-04 06:05:55 -06:00
|
|
|
qid = sxdp->sxdp_queue_id;
|
|
|
|
|
|
|
|
if (flags & XDP_SHARED_UMEM) {
|
2018-05-02 05:01:26 -06:00
|
|
|
struct xdp_sock *umem_xs;
|
|
|
|
struct socket *sock;
|
|
|
|
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY) ||
|
|
|
|
(flags & XDP_USE_NEED_WAKEUP)) {
|
2018-06-04 06:05:55 -06:00
|
|
|
/* Cannot specify flags for shared sockets. */
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:26 -06:00
|
|
|
if (xs->umem) {
|
|
|
|
/* We have already our own. */
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
|
|
|
|
if (IS_ERR(sock)) {
|
|
|
|
err = PTR_ERR(sock);
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
umem_xs = xdp_sk(sock->sk);
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
if (!xsk_is_bound(umem_xs)) {
|
2018-05-02 05:01:26 -06:00
|
|
|
err = -EBADF;
|
|
|
|
sockfd_put(sock);
|
|
|
|
goto out_unlock;
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
}
|
|
|
|
if (umem_xs->dev != dev || umem_xs->queue_id != qid) {
|
2018-05-02 05:01:26 -06:00
|
|
|
err = -EINVAL;
|
|
|
|
sockfd_put(sock);
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
xdp_get_umem(umem_xs->umem);
|
2019-09-04 05:49:11 -06:00
|
|
|
WRITE_ONCE(xs->umem, umem_xs->umem);
|
2018-05-02 05:01:26 -06:00
|
|
|
sockfd_put(sock);
|
|
|
|
} else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_unlock;
|
2018-05-02 05:01:27 -06:00
|
|
|
} else {
|
|
|
|
/* This xsk has its own umem. */
|
2018-06-04 06:05:55 -06:00
|
|
|
err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
|
|
|
|
if (err)
|
|
|
|
goto out_unlock;
|
2018-05-02 05:01:26 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
xs->dev = dev;
|
2018-06-04 06:05:57 -06:00
|
|
|
xs->zc = xs->umem->zc;
|
|
|
|
xs->queue_id = qid;
|
|
|
|
xdp_add_sk_umem(xs->umem, xs);
|
2018-05-02 05:01:26 -06:00
|
|
|
|
|
|
|
out_unlock:
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
if (err) {
|
2018-05-02 05:01:26 -06:00
|
|
|
dev_put(dev);
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
} else {
|
|
|
|
/* Matches smp_rmb() in bind() for shared umem
|
|
|
|
* sockets, and xsk_is_bound().
|
|
|
|
*/
|
|
|
|
smp_wmb();
|
|
|
|
WRITE_ONCE(xs->state, XSK_BOUND);
|
|
|
|
}
|
2018-05-02 05:01:26 -06:00
|
|
|
out_release:
|
|
|
|
mutex_unlock(&xs->mutex);
|
2019-07-08 05:03:44 -06:00
|
|
|
rtnl_unlock();
|
2018-05-02 05:01:26 -06:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
struct xdp_umem_reg_v1 {
|
|
|
|
__u64 addr; /* Start of packet data area */
|
|
|
|
__u64 len; /* Length of packet data area */
|
|
|
|
__u32 chunk_size;
|
|
|
|
__u32 headroom;
|
|
|
|
};
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
static int xsk_setsockopt(struct socket *sock, int level, int optname,
|
|
|
|
char __user *optval, unsigned int optlen)
|
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (level != SOL_XDP)
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
switch (optname) {
|
2018-05-02 05:01:25 -06:00
|
|
|
case XDP_RX_RING:
|
2018-05-02 05:01:32 -06:00
|
|
|
case XDP_TX_RING:
|
2018-05-02 05:01:25 -06:00
|
|
|
{
|
|
|
|
struct xsk_queue **q;
|
|
|
|
int entries;
|
|
|
|
|
|
|
|
if (optlen < sizeof(entries))
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&entries, optval, sizeof(entries)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2019-06-28 02:04:07 -06:00
|
|
|
if (xs->state != XSK_READY) {
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
2018-05-02 05:01:32 -06:00
|
|
|
q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
|
2018-05-02 05:01:25 -06:00
|
|
|
err = xsk_init_queue(entries, q, false);
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
if (!err && optname == XDP_TX_RING)
|
|
|
|
/* Tx needs to be explicitly woken up the first time */
|
|
|
|
xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
|
2018-05-02 05:01:25 -06:00
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return err;
|
|
|
|
}
|
2018-05-02 05:01:23 -06:00
|
|
|
case XDP_UMEM_REG:
|
|
|
|
{
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
size_t mr_size = sizeof(struct xdp_umem_reg);
|
|
|
|
struct xdp_umem_reg mr = {};
|
2018-05-02 05:01:23 -06:00
|
|
|
struct xdp_umem *umem;
|
|
|
|
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-26 20:25:22 -06:00
|
|
|
if (optlen < sizeof(struct xdp_umem_reg_v1))
|
|
|
|
return -EINVAL;
|
|
|
|
else if (optlen < sizeof(mr))
|
|
|
|
mr_size = sizeof(struct xdp_umem_reg_v1);
|
|
|
|
|
|
|
|
if (copy_from_user(&mr, optval, mr_size))
|
2018-05-02 05:01:23 -06:00
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2019-06-28 02:04:07 -06:00
|
|
|
if (xs->state != XSK_READY || xs->umem) {
|
2018-05-22 01:35:02 -06:00
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
2018-05-02 05:01:23 -06:00
|
|
|
|
2018-05-22 01:35:02 -06:00
|
|
|
umem = xdp_umem_create(&mr);
|
|
|
|
if (IS_ERR(umem)) {
|
2018-05-02 05:01:23 -06:00
|
|
|
mutex_unlock(&xs->mutex);
|
2018-05-22 01:35:02 -06:00
|
|
|
return PTR_ERR(umem);
|
2018-05-02 05:01:23 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Make sure umem is ready before it can be seen by others */
|
|
|
|
smp_wmb();
|
2019-09-04 05:49:11 -06:00
|
|
|
WRITE_ONCE(xs->umem, umem);
|
2018-05-02 05:01:23 -06:00
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return 0;
|
|
|
|
}
|
2018-05-02 05:01:24 -06:00
|
|
|
case XDP_UMEM_FILL_RING:
|
2018-05-02 05:01:31 -06:00
|
|
|
case XDP_UMEM_COMPLETION_RING:
|
2018-05-02 05:01:24 -06:00
|
|
|
{
|
|
|
|
struct xsk_queue **q;
|
|
|
|
int entries;
|
|
|
|
|
|
|
|
if (copy_from_user(&entries, optval, sizeof(entries)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
2019-06-28 02:04:07 -06:00
|
|
|
if (xs->state != XSK_READY) {
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
2018-05-22 01:35:02 -06:00
|
|
|
if (!xs->umem) {
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:31 -06:00
|
|
|
q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
|
|
|
|
&xs->umem->cq;
|
2018-05-02 05:01:25 -06:00
|
|
|
err = xsk_init_queue(entries, q, true);
|
2020-05-20 13:20:53 -06:00
|
|
|
if (optname == XDP_UMEM_FILL_RING)
|
|
|
|
xp_set_fq(xs->umem->pool, *q);
|
2018-05-02 05:01:24 -06:00
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
return err;
|
|
|
|
}
|
2018-05-02 05:01:23 -06:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
}
|
|
|
|
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
static void xsk_enter_rxtx_offsets(struct xdp_ring_offset_v1 *ring)
|
|
|
|
{
|
|
|
|
ring->producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
|
|
|
|
ring->consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
|
|
|
|
ring->desc = offsetof(struct xdp_rxtx_ring, desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void xsk_enter_umem_offsets(struct xdp_ring_offset_v1 *ring)
|
|
|
|
{
|
|
|
|
ring->producer = offsetof(struct xdp_umem_ring, ptrs.producer);
|
|
|
|
ring->consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
|
|
|
|
ring->desc = offsetof(struct xdp_umem_ring, desc);
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:35 -06:00
|
|
|
static int xsk_getsockopt(struct socket *sock, int level, int optname,
|
|
|
|
char __user *optval, int __user *optlen)
|
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
int len;
|
|
|
|
|
|
|
|
if (level != SOL_XDP)
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
if (get_user(len, optlen))
|
|
|
|
return -EFAULT;
|
|
|
|
if (len < 0)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
switch (optname) {
|
|
|
|
case XDP_STATISTICS:
|
|
|
|
{
|
|
|
|
struct xdp_statistics stats;
|
|
|
|
|
|
|
|
if (len < sizeof(stats))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
stats.rx_dropped = xs->rx_dropped;
|
|
|
|
stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
|
|
|
|
stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
|
|
|
if (copy_to_user(optval, &stats, sizeof(stats)))
|
|
|
|
return -EFAULT;
|
|
|
|
if (put_user(sizeof(stats), optlen))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 01:34:59 -06:00
|
|
|
case XDP_MMAP_OFFSETS:
|
|
|
|
{
|
|
|
|
struct xdp_mmap_offsets off;
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
struct xdp_mmap_offsets_v1 off_v1;
|
|
|
|
bool flags_supported = true;
|
|
|
|
void *to_copy;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 01:34:59 -06:00
|
|
|
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
if (len < sizeof(off_v1))
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 01:34:59 -06:00
|
|
|
return -EINVAL;
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
else if (len < sizeof(off))
|
|
|
|
flags_supported = false;
|
|
|
|
|
|
|
|
if (flags_supported) {
|
|
|
|
/* xdp_ring_offset is identical to xdp_ring_offset_v1
|
|
|
|
* except for the flags field added to the end.
|
|
|
|
*/
|
|
|
|
xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
|
|
|
|
&off.rx);
|
|
|
|
xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
|
|
|
|
&off.tx);
|
|
|
|
xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
|
|
|
|
&off.fr);
|
|
|
|
xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
|
|
|
|
&off.cr);
|
|
|
|
off.rx.flags = offsetof(struct xdp_rxtx_ring,
|
|
|
|
ptrs.flags);
|
|
|
|
off.tx.flags = offsetof(struct xdp_rxtx_ring,
|
|
|
|
ptrs.flags);
|
|
|
|
off.fr.flags = offsetof(struct xdp_umem_ring,
|
|
|
|
ptrs.flags);
|
|
|
|
off.cr.flags = offsetof(struct xdp_umem_ring,
|
|
|
|
ptrs.flags);
|
|
|
|
|
|
|
|
len = sizeof(off);
|
|
|
|
to_copy = &off;
|
|
|
|
} else {
|
|
|
|
xsk_enter_rxtx_offsets(&off_v1.rx);
|
|
|
|
xsk_enter_rxtx_offsets(&off_v1.tx);
|
|
|
|
xsk_enter_umem_offsets(&off_v1.fr);
|
|
|
|
xsk_enter_umem_offsets(&off_v1.cr);
|
|
|
|
|
|
|
|
len = sizeof(off_v1);
|
|
|
|
to_copy = &off_v1;
|
|
|
|
}
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 01:34:59 -06:00
|
|
|
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 01:27:17 -06:00
|
|
|
if (copy_to_user(optval, to_copy, len))
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 01:34:59 -06:00
|
|
|
return -EFAULT;
|
|
|
|
if (put_user(len, optlen))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2019-06-26 08:35:25 -06:00
|
|
|
case XDP_OPTIONS:
|
|
|
|
{
|
|
|
|
struct xdp_options opts = {};
|
|
|
|
|
|
|
|
if (len < sizeof(opts))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
if (xs->zc)
|
|
|
|
opts.flags |= XDP_OPTIONS_ZEROCOPY;
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
|
|
|
|
len = sizeof(opts);
|
|
|
|
if (copy_to_user(optval, &opts, len))
|
|
|
|
return -EFAULT;
|
|
|
|
if (put_user(len, optlen))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2018-05-02 05:01:35 -06:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:24 -06:00
|
|
|
static int xsk_mmap(struct file *file, struct socket *sock,
|
|
|
|
struct vm_area_struct *vma)
|
|
|
|
{
|
2018-06-07 07:37:34 -06:00
|
|
|
loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
|
2018-05-02 05:01:24 -06:00
|
|
|
unsigned long size = vma->vm_end - vma->vm_start;
|
|
|
|
struct xdp_sock *xs = xdp_sk(sock->sk);
|
|
|
|
struct xsk_queue *q = NULL;
|
2018-05-22 01:35:01 -06:00
|
|
|
struct xdp_umem *umem;
|
2018-05-02 05:01:24 -06:00
|
|
|
unsigned long pfn;
|
|
|
|
struct page *qpg;
|
|
|
|
|
xsk: use state member for socket synchronization
Prior the state variable was introduced by Ilya, the dev member was
used to determine whether the socket was bound or not. However, when
dev was read, proper SMP barriers and READ_ONCE were missing. In order
to address the missing barriers and READ_ONCE, we start using the
state variable as a point of synchronization. The state member
read/write is paired with proper SMP barriers, and from this follows
that the members described above does not need READ_ONCE if used in
conjunction with state check.
In all syscalls and the xsk_rcv path we check if state is
XSK_BOUND. If that is the case we do a SMP read barrier, and this
implies that the dev, umem and all rings are correctly setup. Note
that no READ_ONCE are needed for these variable if used when state is
XSK_BOUND (plus the read barrier).
To summarize: The members struct xdp_sock members dev, queue_id, umem,
fq, cq, tx, rx, and state were read lock-less, with incorrect barriers
and missing {READ, WRITE}_ONCE. Now, umem, fq, cq, tx, rx, and state
are read lock-less. When these members are updated, WRITE_ONCE is
used. When read, READ_ONCE are only used when read outside the control
mutex (e.g. mmap) or, not synchronized with the state member
(XSK_BOUND plus smp_rmb())
Note that dev and queue_id do not need a WRITE_ONCE or READ_ONCE, due
to the introduce state synchronization (XSK_BOUND plus smp_rmb()).
Introducing the state check also fixes a race, found by syzcaller, in
xsk_poll() where umem could be accessed when stale.
Suggested-by: Hillf Danton <hdanton@sina.com>
Reported-by: syzbot+c82697e3043781e08802@syzkaller.appspotmail.com
Fixes: 77cd0d7b3f25 ("xsk: add support for need_wakeup flag in AF_XDP rings")
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-09-04 05:49:12 -06:00
|
|
|
if (READ_ONCE(xs->state) != XSK_READY)
|
2019-06-28 02:04:07 -06:00
|
|
|
return -EBUSY;
|
|
|
|
|
2018-05-02 05:01:25 -06:00
|
|
|
if (offset == XDP_PGOFF_RX_RING) {
|
2018-05-22 01:35:01 -06:00
|
|
|
q = READ_ONCE(xs->rx);
|
2018-05-02 05:01:32 -06:00
|
|
|
} else if (offset == XDP_PGOFF_TX_RING) {
|
2018-05-22 01:35:01 -06:00
|
|
|
q = READ_ONCE(xs->tx);
|
2018-05-02 05:01:25 -06:00
|
|
|
} else {
|
2018-05-22 01:35:01 -06:00
|
|
|
umem = READ_ONCE(xs->umem);
|
|
|
|
if (!umem)
|
2018-05-02 05:01:25 -06:00
|
|
|
return -EINVAL;
|
2018-05-02 05:01:24 -06:00
|
|
|
|
2019-02-08 06:13:50 -07:00
|
|
|
/* Matches the smp_wmb() in XDP_UMEM_REG */
|
|
|
|
smp_rmb();
|
2018-05-02 05:01:25 -06:00
|
|
|
if (offset == XDP_UMEM_PGOFF_FILL_RING)
|
2018-05-22 01:35:01 -06:00
|
|
|
q = READ_ONCE(umem->fq);
|
2018-05-02 05:01:31 -06:00
|
|
|
else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
|
2018-05-22 01:35:01 -06:00
|
|
|
q = READ_ONCE(umem->cq);
|
2018-05-02 05:01:25 -06:00
|
|
|
}
|
2018-05-02 05:01:24 -06:00
|
|
|
|
|
|
|
if (!q)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2019-02-08 06:13:50 -07:00
|
|
|
/* Matches the smp_wmb() in xsk_init_queue */
|
|
|
|
smp_rmb();
|
2018-05-02 05:01:24 -06:00
|
|
|
qpg = virt_to_head_page(q->ring);
|
2019-09-23 16:34:25 -06:00
|
|
|
if (size > page_size(qpg))
|
2018-05-02 05:01:24 -06:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
|
|
|
|
return remap_pfn_range(vma, vma->vm_start, pfn,
|
|
|
|
size, vma->vm_page_prot);
|
|
|
|
}
|
|
|
|
|
2019-06-28 02:04:07 -06:00
|
|
|
static int xsk_notifier(struct notifier_block *this,
|
|
|
|
unsigned long msg, void *ptr)
|
|
|
|
{
|
|
|
|
struct net_device *dev = netdev_notifier_info_to_dev(ptr);
|
|
|
|
struct net *net = dev_net(dev);
|
|
|
|
struct sock *sk;
|
|
|
|
|
|
|
|
switch (msg) {
|
|
|
|
case NETDEV_UNREGISTER:
|
|
|
|
mutex_lock(&net->xdp.lock);
|
|
|
|
sk_for_each(sk, &net->xdp.list) {
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
mutex_lock(&xs->mutex);
|
|
|
|
if (xs->dev == dev) {
|
|
|
|
sk->sk_err = ENETDOWN;
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
|
|
|
sk->sk_error_report(sk);
|
|
|
|
|
|
|
|
xsk_unbind_dev(xs);
|
|
|
|
|
|
|
|
/* Clear device references in umem. */
|
|
|
|
xdp_umem_clear_dev(xs->umem);
|
|
|
|
}
|
|
|
|
mutex_unlock(&xs->mutex);
|
|
|
|
}
|
|
|
|
mutex_unlock(&net->xdp.lock);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
static struct proto xsk_proto = {
|
|
|
|
.name = "XDP",
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.obj_size = sizeof(struct xdp_sock),
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct proto_ops xsk_proto_ops = {
|
2018-05-18 06:00:24 -06:00
|
|
|
.family = PF_XDP,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.release = xsk_release,
|
|
|
|
.bind = xsk_bind,
|
|
|
|
.connect = sock_no_connect,
|
|
|
|
.socketpair = sock_no_socketpair,
|
|
|
|
.accept = sock_no_accept,
|
|
|
|
.getname = sock_no_getname,
|
2018-06-28 10:43:44 -06:00
|
|
|
.poll = xsk_poll,
|
2018-05-18 06:00:24 -06:00
|
|
|
.ioctl = sock_no_ioctl,
|
|
|
|
.listen = sock_no_listen,
|
|
|
|
.shutdown = sock_no_shutdown,
|
|
|
|
.setsockopt = xsk_setsockopt,
|
|
|
|
.getsockopt = xsk_getsockopt,
|
|
|
|
.sendmsg = xsk_sendmsg,
|
|
|
|
.recvmsg = sock_no_recvmsg,
|
|
|
|
.mmap = xsk_mmap,
|
|
|
|
.sendpage = sock_no_sendpage,
|
2018-05-02 05:01:23 -06:00
|
|
|
};
|
|
|
|
|
2019-02-21 05:07:38 -07:00
|
|
|
static void xsk_destruct(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct xdp_sock *xs = xdp_sk(sk);
|
|
|
|
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
|
|
|
return;
|
|
|
|
|
|
|
|
xdp_put_umem(xs->umem);
|
|
|
|
|
|
|
|
sk_refcnt_debug_dec(sk);
|
|
|
|
}
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
static int xsk_create(struct net *net, struct socket *sock, int protocol,
|
|
|
|
int kern)
|
|
|
|
{
|
|
|
|
struct sock *sk;
|
|
|
|
struct xdp_sock *xs;
|
|
|
|
|
|
|
|
if (!ns_capable(net->user_ns, CAP_NET_RAW))
|
|
|
|
return -EPERM;
|
|
|
|
if (sock->type != SOCK_RAW)
|
|
|
|
return -ESOCKTNOSUPPORT;
|
|
|
|
|
|
|
|
if (protocol)
|
|
|
|
return -EPROTONOSUPPORT;
|
|
|
|
|
|
|
|
sock->state = SS_UNCONNECTED;
|
|
|
|
|
|
|
|
sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
|
|
|
|
if (!sk)
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
sock->ops = &xsk_proto_ops;
|
|
|
|
|
|
|
|
sock_init_data(sock, sk);
|
|
|
|
|
|
|
|
sk->sk_family = PF_XDP;
|
|
|
|
|
2019-02-21 05:07:38 -07:00
|
|
|
sk->sk_destruct = xsk_destruct;
|
|
|
|
sk_refcnt_debug_inc(sk);
|
|
|
|
|
2018-10-08 11:40:16 -06:00
|
|
|
sock_set_flag(sk, SOCK_RCU_FREE);
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
xs = xdp_sk(sk);
|
2019-06-28 02:04:07 -06:00
|
|
|
xs->state = XSK_READY;
|
2018-05-02 05:01:23 -06:00
|
|
|
mutex_init(&xs->mutex);
|
2019-07-03 06:09:16 -06:00
|
|
|
spin_lock_init(&xs->rx_lock);
|
2018-06-29 01:48:20 -06:00
|
|
|
spin_lock_init(&xs->tx_completion_lock);
|
2018-05-02 05:01:23 -06:00
|
|
|
|
2019-08-15 03:30:13 -06:00
|
|
|
INIT_LIST_HEAD(&xs->map_list);
|
|
|
|
spin_lock_init(&xs->map_list_lock);
|
|
|
|
|
2019-01-24 11:59:37 -07:00
|
|
|
mutex_lock(&net->xdp.lock);
|
|
|
|
sk_add_node_rcu(sk, &net->xdp.list);
|
|
|
|
mutex_unlock(&net->xdp.lock);
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
local_bh_disable();
|
|
|
|
sock_prot_inuse_add(net, &xsk_proto, 1);
|
|
|
|
local_bh_enable();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct net_proto_family xsk_family_ops = {
|
|
|
|
.family = PF_XDP,
|
|
|
|
.create = xsk_create,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
2019-06-28 02:04:07 -06:00
|
|
|
static struct notifier_block xsk_netdev_notifier = {
|
|
|
|
.notifier_call = xsk_notifier,
|
|
|
|
};
|
|
|
|
|
2019-01-24 11:59:37 -07:00
|
|
|
static int __net_init xsk_net_init(struct net *net)
|
|
|
|
{
|
|
|
|
mutex_init(&net->xdp.lock);
|
|
|
|
INIT_HLIST_HEAD(&net->xdp.list);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __net_exit xsk_net_exit(struct net *net)
|
|
|
|
{
|
|
|
|
WARN_ON_ONCE(!hlist_empty(&net->xdp.list));
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations xsk_net_ops = {
|
|
|
|
.init = xsk_net_init,
|
|
|
|
.exit = xsk_net_exit,
|
|
|
|
};
|
|
|
|
|
2018-05-02 05:01:23 -06:00
|
|
|
static int __init xsk_init(void)
|
|
|
|
{
|
2019-12-18 23:10:02 -07:00
|
|
|
int err, cpu;
|
2018-05-02 05:01:23 -06:00
|
|
|
|
|
|
|
err = proto_register(&xsk_proto, 0 /* no slab */);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
err = sock_register(&xsk_family_ops);
|
|
|
|
if (err)
|
|
|
|
goto out_proto;
|
|
|
|
|
2019-01-24 11:59:37 -07:00
|
|
|
err = register_pernet_subsys(&xsk_net_ops);
|
|
|
|
if (err)
|
|
|
|
goto out_sk;
|
2019-06-28 02:04:07 -06:00
|
|
|
|
|
|
|
err = register_netdevice_notifier(&xsk_netdev_notifier);
|
|
|
|
if (err)
|
|
|
|
goto out_pernet;
|
|
|
|
|
2019-12-18 23:10:02 -07:00
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
INIT_LIST_HEAD(&per_cpu(xskmap_flush_list, cpu));
|
2018-05-02 05:01:23 -06:00
|
|
|
return 0;
|
|
|
|
|
2019-06-28 02:04:07 -06:00
|
|
|
out_pernet:
|
|
|
|
unregister_pernet_subsys(&xsk_net_ops);
|
2019-01-24 11:59:37 -07:00
|
|
|
out_sk:
|
|
|
|
sock_unregister(PF_XDP);
|
2018-05-02 05:01:23 -06:00
|
|
|
out_proto:
|
|
|
|
proto_unregister(&xsk_proto);
|
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
fs_initcall(xsk_init);
|