redonkable
/
alistair23-linux
1
0
Fork 0
Code Releases Activity

xsk: Enable sharing of dma mappings 

Enable the sharing of dma mappings by moving them out from the buffer
pool. Instead we put each dma mapped umem region in a list in the umem
structure. If dma has already been mapped for this umem and device, it
is not mapped again and the existing dma mappings are reused.

Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Björn Töpel <bjorn.topel@intel.com>
Link: https://lore.kernel.org/bpf/1598603189-32145-9-git-send-email-magnus.karlsson@intel.com
zero-sugar-mainline-defconfig
Magnus Karlsson 2020-08-28 10:26:22 +02:00 committed by Daniel Borkmann
parent 7f7ffa4e9c
commit 921b68692a
4 changed files with 156 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/net/xdp_sock.h
View File

@ -30,6 +30,7 @@ struct xdp_umem {
u8 flags;
int id;
bool zc;
struct list_head xsk_dma_list;
};
struct xsk_map {

13
include/net/xsk_buff_pool.h
View File

@ -28,10 +28,23 @@ struct xdp_buff_xsk {
struct list_head free_list_node;
};
struct xsk_dma_map {
dma_addr_t *dma_pages;
struct device *dev;
struct net_device *netdev;
refcount_t users;
struct list_head list; /* Protected by the RTNL_LOCK */
u32 dma_pages_cnt;
bool dma_need_sync;
};
struct xsk_buff_pool {
struct xsk_queue *fq;
struct xsk_queue *cq;
struct list_head free_list;
/* For performance reasons, each buff pool has its own array of dma_pages
* even when they are identical.
*/
dma_addr_t *dma_pages;
struct xdp_buff_xsk *heads;
u64 chunk_mask;

1
net/xdp/xdp_umem.c
View File

@ -198,6 +198,7 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
umem->user = NULL;
umem->flags = mr->flags;
INIT_LIST_HEAD(&umem->xsk_dma_list);
refcount_set(&umem->users, 1);
err = xdp_umem_account_pages(umem);

183
net/xdp/xsk_buff_pool.c
View File

@ -104,6 +104,25 @@ void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
}
EXPORT_SYMBOL(xp_set_rxq_info);
static void xp_disable_drv_zc(struct xsk_buff_pool *pool)
{
struct netdev_bpf bpf;
int err;
ASSERT_RTNL();
if (pool->umem->zc) {
bpf.command = XDP_SETUP_XSK_POOL;
bpf.xsk.pool = NULL;
bpf.xsk.queue_id = pool->queue_id;
err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);
if (err)
WARN(1, "Failed to disable zero-copy!\n");
}
}
int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
u16 queue_id, u16 flags)
{
@ -122,6 +141,8 @@ int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
if (xsk_get_pool_from_qid(netdev, queue_id))
return -EBUSY;
pool->netdev = netdev;
pool->queue_id = queue_id;
err = xsk_reg_pool_at_qid(netdev, pool, queue_id);
if (err)
return err;
@ -155,11 +176,15 @@ int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
if (err)
goto err_unreg_pool;
pool->netdev = netdev;
pool->queue_id = queue_id;
if (!pool->dma_pages) {
WARN(1, "Driver did not DMA map zero-copy buffers");
goto err_unreg_xsk;
}
pool->umem->zc = true;
return 0;
err_unreg_xsk:
xp_disable_drv_zc(pool);
err_unreg_pool:
if (!force_zc)
err = 0; /* fallback to copy mode */
@ -170,25 +195,10 @@ err_unreg_pool:
void xp_clear_dev(struct xsk_buff_pool *pool)
{
struct netdev_bpf bpf;
int err;
ASSERT_RTNL();
if (!pool->netdev)
return;
if (pool->umem->zc) {
bpf.command = XDP_SETUP_XSK_POOL;
bpf.xsk.pool = NULL;
bpf.xsk.queue_id = pool->queue_id;
err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);
if (err)
WARN(1, "Failed to disable zero-copy!\n");
}
xp_disable_drv_zc(pool);
xsk_clear_pool_at_qid(pool->netdev, pool->queue_id);
dev_put(pool->netdev);
pool->netdev = NULL;
@ -233,70 +243,159 @@ void xp_put_pool(struct xsk_buff_pool *pool)
}
}
void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool)
{
struct xsk_dma_map *dma_map;
list_for_each_entry(dma_map, &pool->umem->xsk_dma_list, list) {
if (dma_map->netdev == pool->netdev)
return dma_map;
}
return NULL;
}
static struct xsk_dma_map *xp_create_dma_map(struct device *dev, struct net_device *netdev,
u32 nr_pages, struct xdp_umem *umem)
{
struct xsk_dma_map *dma_map;
dma_map = kzalloc(sizeof(*dma_map), GFP_KERNEL);
if (!dma_map)
return NULL;
dma_map->dma_pages = kvcalloc(nr_pages, sizeof(*dma_map->dma_pages), GFP_KERNEL);
if (!dma_map) {
kfree(dma_map);
return NULL;
}
dma_map->netdev = netdev;
dma_map->dev = dev;
dma_map->dma_need_sync = false;
dma_map->dma_pages_cnt = nr_pages;
refcount_set(&dma_map->users, 0);
list_add(&dma_map->list, &umem->xsk_dma_list);
return dma_map;
}
static void xp_destroy_dma_map(struct xsk_dma_map *dma_map)
{
list_del(&dma_map->list);
kvfree(dma_map->dma_pages);
kfree(dma_map);
}
static void __xp_dma_unmap(struct xsk_dma_map *dma_map, unsigned long attrs)
{
dma_addr_t *dma;
u32 i;
if (pool->dma_pages_cnt == 0)
return;
for (i = 0; i < pool->dma_pages_cnt; i++) {
dma = &pool->dma_pages[i];
for (i = 0; i < dma_map->dma_pages_cnt; i++) {
dma = &dma_map->dma_pages[i];
if (*dma) {
dma_unmap_page_attrs(pool->dev, *dma, PAGE_SIZE,
dma_unmap_page_attrs(dma_map->dev, *dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, attrs);
*dma = 0;
}
}
xp_destroy_dma_map(dma_map);
}
void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
{
struct xsk_dma_map *dma_map;
if (pool->dma_pages_cnt == 0)
return;
dma_map = xp_find_dma_map(pool);
if (!dma_map) {
WARN(1, "Could not find dma_map for device");
return;
}
if (!refcount_dec_and_test(&dma_map->users))
return;
__xp_dma_unmap(dma_map, attrs);
kvfree(pool->dma_pages);
pool->dma_pages_cnt = 0;
pool->dev = NULL;
}
EXPORT_SYMBOL(xp_dma_unmap);
static void xp_check_dma_contiguity(struct xsk_buff_pool *pool)
static void xp_check_dma_contiguity(struct xsk_dma_map *dma_map)
{
u32 i;
for (i = 0; i < pool->dma_pages_cnt - 1; i++) {
if (pool->dma_pages[i] + PAGE_SIZE == pool->dma_pages[i + 1])
pool->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
for (i = 0; i < dma_map->dma_pages_cnt - 1; i++) {
if (dma_map->dma_pages[i] + PAGE_SIZE == dma_map->dma_pages[i + 1])
dma_map->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
else
pool->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
dma_map->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
}
}
static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map)
{
pool->dma_pages = kvcalloc(dma_map->dma_pages_cnt, sizeof(*pool->dma_pages), GFP_KERNEL);
if (!pool->dma_pages)
return -ENOMEM;
pool->dev = dma_map->dev;
pool->dma_pages_cnt = dma_map->dma_pages_cnt;
pool->dma_need_sync = dma_map->dma_need_sync;
refcount_inc(&dma_map->users);
memcpy(pool->dma_pages, dma_map->dma_pages,
pool->dma_pages_cnt * sizeof(*pool->dma_pages));
return 0;
}
int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
unsigned long attrs, struct page **pages, u32 nr_pages)
{
struct xsk_dma_map *dma_map;
dma_addr_t dma;
int err;
u32 i;
pool->dma_pages = kvcalloc(nr_pages, sizeof(*pool->dma_pages),
GFP_KERNEL);
if (!pool->dma_pages)
dma_map = xp_find_dma_map(pool);
if (dma_map) {
err = xp_init_dma_info(pool, dma_map);
if (err)
return err;
return 0;
}
dma_map = xp_create_dma_map(dev, pool->netdev, nr_pages, pool->umem);
if (!dma_map)
return -ENOMEM;
pool->dev = dev;
pool->dma_pages_cnt = nr_pages;
pool->dma_need_sync = false;
for (i = 0; i < pool->dma_pages_cnt; i++) {
for (i = 0; i < dma_map->dma_pages_cnt; i++) {
dma = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE,
DMA_BIDIRECTIONAL, attrs);
if (dma_mapping_error(dev, dma)) {
xp_dma_unmap(pool, attrs);
__xp_dma_unmap(dma_map, attrs);
return -ENOMEM;
}
if (dma_need_sync(dev, dma))
pool->dma_need_sync = true;
pool->dma_pages[i] = dma;
dma_map->dma_need_sync = true;
dma_map->dma_pages[i] = dma;
}
if (pool->unaligned)
xp_check_dma_contiguity(pool);
xp_check_dma_contiguity(dma_map);
err = xp_init_dma_info(pool, dma_map);
if (err) {
__xp_dma_unmap(dma_map, attrs);
return err;
}
return 0;
}
EXPORT_SYMBOL(xp_dma_map);