xsk: Enable sharing of dma mappings
Enable the sharing of dma mappings by moving them out from the buffer pool. Instead we put each dma mapped umem region in a list in the umem structure. If dma has already been mapped for this umem and device, it is not mapped again and the existing dma mappings are reused. Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> Acked-by: Björn Töpel <bjorn.topel@intel.com> Link: https://lore.kernel.org/bpf/1598603189-32145-9-git-send-email-magnus.karlsson@intel.com
This commit is contained in:
parent
7f7ffa4e9c
commit
921b68692a
|
@ -30,6 +30,7 @@ struct xdp_umem {
|
|||
u8 flags;
|
||||
int id;
|
||||
bool zc;
|
||||
struct list_head xsk_dma_list;
|
||||
};
|
||||
|
||||
struct xsk_map {
|
||||
|
|
|
@ -28,10 +28,23 @@ struct xdp_buff_xsk {
|
|||
struct list_head free_list_node;
|
||||
};
|
||||
|
||||
struct xsk_dma_map {
|
||||
dma_addr_t *dma_pages;
|
||||
struct device *dev;
|
||||
struct net_device *netdev;
|
||||
refcount_t users;
|
||||
struct list_head list; /* Protected by the RTNL_LOCK */
|
||||
u32 dma_pages_cnt;
|
||||
bool dma_need_sync;
|
||||
};
|
||||
|
||||
struct xsk_buff_pool {
|
||||
struct xsk_queue *fq;
|
||||
struct xsk_queue *cq;
|
||||
struct list_head free_list;
|
||||
/* For performance reasons, each buff pool has its own array of dma_pages
|
||||
* even when they are identical.
|
||||
*/
|
||||
dma_addr_t *dma_pages;
|
||||
struct xdp_buff_xsk *heads;
|
||||
u64 chunk_mask;
|
||||
|
|
|
@ -198,6 +198,7 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
|
|||
umem->user = NULL;
|
||||
umem->flags = mr->flags;
|
||||
|
||||
INIT_LIST_HEAD(&umem->xsk_dma_list);
|
||||
refcount_set(&umem->users, 1);
|
||||
|
||||
err = xdp_umem_account_pages(umem);
|
||||
|
|
|
@ -104,6 +104,25 @@ void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
|
|||
}
|
||||
EXPORT_SYMBOL(xp_set_rxq_info);
|
||||
|
||||
static void xp_disable_drv_zc(struct xsk_buff_pool *pool)
|
||||
{
|
||||
struct netdev_bpf bpf;
|
||||
int err;
|
||||
|
||||
ASSERT_RTNL();
|
||||
|
||||
if (pool->umem->zc) {
|
||||
bpf.command = XDP_SETUP_XSK_POOL;
|
||||
bpf.xsk.pool = NULL;
|
||||
bpf.xsk.queue_id = pool->queue_id;
|
||||
|
||||
err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);
|
||||
|
||||
if (err)
|
||||
WARN(1, "Failed to disable zero-copy!\n");
|
||||
}
|
||||
}
|
||||
|
||||
int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
|
||||
u16 queue_id, u16 flags)
|
||||
{
|
||||
|
@ -122,6 +141,8 @@ int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
|
|||
if (xsk_get_pool_from_qid(netdev, queue_id))
|
||||
return -EBUSY;
|
||||
|
||||
pool->netdev = netdev;
|
||||
pool->queue_id = queue_id;
|
||||
err = xsk_reg_pool_at_qid(netdev, pool, queue_id);
|
||||
if (err)
|
||||
return err;
|
||||
|
@ -155,11 +176,15 @@ int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
|
|||
if (err)
|
||||
goto err_unreg_pool;
|
||||
|
||||
pool->netdev = netdev;
|
||||
pool->queue_id = queue_id;
|
||||
if (!pool->dma_pages) {
|
||||
WARN(1, "Driver did not DMA map zero-copy buffers");
|
||||
goto err_unreg_xsk;
|
||||
}
|
||||
pool->umem->zc = true;
|
||||
return 0;
|
||||
|
||||
err_unreg_xsk:
|
||||
xp_disable_drv_zc(pool);
|
||||
err_unreg_pool:
|
||||
if (!force_zc)
|
||||
err = 0; /* fallback to copy mode */
|
||||
|
@ -170,25 +195,10 @@ err_unreg_pool:
|
|||
|
||||
void xp_clear_dev(struct xsk_buff_pool *pool)
|
||||
{
|
||||
struct netdev_bpf bpf;
|
||||
int err;
|
||||
|
||||
ASSERT_RTNL();
|
||||
|
||||
if (!pool->netdev)
|
||||
return;
|
||||
|
||||
if (pool->umem->zc) {
|
||||
bpf.command = XDP_SETUP_XSK_POOL;
|
||||
bpf.xsk.pool = NULL;
|
||||
bpf.xsk.queue_id = pool->queue_id;
|
||||
|
||||
err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);
|
||||
|
||||
if (err)
|
||||
WARN(1, "Failed to disable zero-copy!\n");
|
||||
}
|
||||
|
||||
xp_disable_drv_zc(pool);
|
||||
xsk_clear_pool_at_qid(pool->netdev, pool->queue_id);
|
||||
dev_put(pool->netdev);
|
||||
pool->netdev = NULL;
|
||||
|
@ -233,70 +243,159 @@ void xp_put_pool(struct xsk_buff_pool *pool)
|
|||
}
|
||||
}
|
||||
|
||||
void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
|
||||
static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool)
|
||||
{
|
||||
struct xsk_dma_map *dma_map;
|
||||
|
||||
list_for_each_entry(dma_map, &pool->umem->xsk_dma_list, list) {
|
||||
if (dma_map->netdev == pool->netdev)
|
||||
return dma_map;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct xsk_dma_map *xp_create_dma_map(struct device *dev, struct net_device *netdev,
|
||||
u32 nr_pages, struct xdp_umem *umem)
|
||||
{
|
||||
struct xsk_dma_map *dma_map;
|
||||
|
||||
dma_map = kzalloc(sizeof(*dma_map), GFP_KERNEL);
|
||||
if (!dma_map)
|
||||
return NULL;
|
||||
|
||||
dma_map->dma_pages = kvcalloc(nr_pages, sizeof(*dma_map->dma_pages), GFP_KERNEL);
|
||||
if (!dma_map) {
|
||||
kfree(dma_map);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
dma_map->netdev = netdev;
|
||||
dma_map->dev = dev;
|
||||
dma_map->dma_need_sync = false;
|
||||
dma_map->dma_pages_cnt = nr_pages;
|
||||
refcount_set(&dma_map->users, 0);
|
||||
list_add(&dma_map->list, &umem->xsk_dma_list);
|
||||
return dma_map;
|
||||
}
|
||||
|
||||
static void xp_destroy_dma_map(struct xsk_dma_map *dma_map)
|
||||
{
|
||||
list_del(&dma_map->list);
|
||||
kvfree(dma_map->dma_pages);
|
||||
kfree(dma_map);
|
||||
}
|
||||
|
||||
static void __xp_dma_unmap(struct xsk_dma_map *dma_map, unsigned long attrs)
|
||||
{
|
||||
dma_addr_t *dma;
|
||||
u32 i;
|
||||
|
||||
if (pool->dma_pages_cnt == 0)
|
||||
return;
|
||||
|
||||
for (i = 0; i < pool->dma_pages_cnt; i++) {
|
||||
dma = &pool->dma_pages[i];
|
||||
for (i = 0; i < dma_map->dma_pages_cnt; i++) {
|
||||
dma = &dma_map->dma_pages[i];
|
||||
if (*dma) {
|
||||
dma_unmap_page_attrs(pool->dev, *dma, PAGE_SIZE,
|
||||
dma_unmap_page_attrs(dma_map->dev, *dma, PAGE_SIZE,
|
||||
DMA_BIDIRECTIONAL, attrs);
|
||||
*dma = 0;
|
||||
}
|
||||
}
|
||||
|
||||
xp_destroy_dma_map(dma_map);
|
||||
}
|
||||
|
||||
void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
|
||||
{
|
||||
struct xsk_dma_map *dma_map;
|
||||
|
||||
if (pool->dma_pages_cnt == 0)
|
||||
return;
|
||||
|
||||
dma_map = xp_find_dma_map(pool);
|
||||
if (!dma_map) {
|
||||
WARN(1, "Could not find dma_map for device");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!refcount_dec_and_test(&dma_map->users))
|
||||
return;
|
||||
|
||||
__xp_dma_unmap(dma_map, attrs);
|
||||
kvfree(pool->dma_pages);
|
||||
pool->dma_pages_cnt = 0;
|
||||
pool->dev = NULL;
|
||||
}
|
||||
EXPORT_SYMBOL(xp_dma_unmap);
|
||||
|
||||
static void xp_check_dma_contiguity(struct xsk_buff_pool *pool)
|
||||
static void xp_check_dma_contiguity(struct xsk_dma_map *dma_map)
|
||||
{
|
||||
u32 i;
|
||||
|
||||
for (i = 0; i < pool->dma_pages_cnt - 1; i++) {
|
||||
if (pool->dma_pages[i] + PAGE_SIZE == pool->dma_pages[i + 1])
|
||||
pool->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
|
||||
for (i = 0; i < dma_map->dma_pages_cnt - 1; i++) {
|
||||
if (dma_map->dma_pages[i] + PAGE_SIZE == dma_map->dma_pages[i + 1])
|
||||
dma_map->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
|
||||
else
|
||||
pool->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
|
||||
dma_map->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
|
||||
}
|
||||
}
|
||||
|
||||
static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map)
|
||||
{
|
||||
pool->dma_pages = kvcalloc(dma_map->dma_pages_cnt, sizeof(*pool->dma_pages), GFP_KERNEL);
|
||||
if (!pool->dma_pages)
|
||||
return -ENOMEM;
|
||||
|
||||
pool->dev = dma_map->dev;
|
||||
pool->dma_pages_cnt = dma_map->dma_pages_cnt;
|
||||
pool->dma_need_sync = dma_map->dma_need_sync;
|
||||
refcount_inc(&dma_map->users);
|
||||
memcpy(pool->dma_pages, dma_map->dma_pages,
|
||||
pool->dma_pages_cnt * sizeof(*pool->dma_pages));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
|
||||
unsigned long attrs, struct page **pages, u32 nr_pages)
|
||||
{
|
||||
struct xsk_dma_map *dma_map;
|
||||
dma_addr_t dma;
|
||||
int err;
|
||||
u32 i;
|
||||
|
||||
pool->dma_pages = kvcalloc(nr_pages, sizeof(*pool->dma_pages),
|
||||
GFP_KERNEL);
|
||||
if (!pool->dma_pages)
|
||||
dma_map = xp_find_dma_map(pool);
|
||||
if (dma_map) {
|
||||
err = xp_init_dma_info(pool, dma_map);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
dma_map = xp_create_dma_map(dev, pool->netdev, nr_pages, pool->umem);
|
||||
if (!dma_map)
|
||||
return -ENOMEM;
|
||||
|
||||
pool->dev = dev;
|
||||
pool->dma_pages_cnt = nr_pages;
|
||||
pool->dma_need_sync = false;
|
||||
|
||||
for (i = 0; i < pool->dma_pages_cnt; i++) {
|
||||
for (i = 0; i < dma_map->dma_pages_cnt; i++) {
|
||||
dma = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE,
|
||||
DMA_BIDIRECTIONAL, attrs);
|
||||
if (dma_mapping_error(dev, dma)) {
|
||||
xp_dma_unmap(pool, attrs);
|
||||
__xp_dma_unmap(dma_map, attrs);
|
||||
return -ENOMEM;
|
||||
}
|
||||
if (dma_need_sync(dev, dma))
|
||||
pool->dma_need_sync = true;
|
||||
pool->dma_pages[i] = dma;
|
||||
dma_map->dma_need_sync = true;
|
||||
dma_map->dma_pages[i] = dma;
|
||||
}
|
||||
|
||||
if (pool->unaligned)
|
||||
xp_check_dma_contiguity(pool);
|
||||
xp_check_dma_contiguity(dma_map);
|
||||
|
||||
err = xp_init_dma_info(pool, dma_map);
|
||||
if (err) {
|
||||
__xp_dma_unmap(dma_map, attrs);
|
||||
return err;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(xp_dma_map);
|
||||
|
|
Loading…
Reference in a new issue