![Adit Ranadive](/assets/img/avatar_default.png)
This patch series adds a driver for a paravirtual RDMA device. The device is developed for VMware's Virtual Machines and allows existing RDMA applications to continue to use existing Verbs API when deployed in VMs on ESXi. We recently did a presentation in the OFA Workshop [1] regarding this device. Description and RDMA Support ============================ The virtual device is exposed as a dual function PCIe device. One part is a virtual network device (VMXNet3) which provides networking properties like MAC, IP addresses to the RDMA part of the device. The networking properties are used to register GIDs required by RDMA applications to communicate. These patches add support and the all required infrastructure for letting applications use such a device. We support the mandatory Verbs API as well as the base memory management extensions (Local Inv, Send with Inv and Fast Register Work Requests). We currently support both Reliable Connected and Unreliable Datagram QPs but do not support Shared Receive Queues (SRQs). Also, we support the following types of Work Requests: o Send/Receive (with or without Immediate Data) o RDMA Write (with or without Immediate Data) o RDMA Read o Local Invalidate o Send with Invalidate o Fast Register Work Requests This version only adds support for version 1 of RoCE. We will add RoCEv2 support in a future patch. We do support registration of both MAC-based and IP-based GIDs. I have also created a git tree for our user-level driver [2]. Testing ======= We have tested this internally for various types of Guest OS - Red Hat, Centos, Ubuntu 12.04/14.04/16.04, Oracle Enterprise Linux, SLES 12 using backported versions of this driver. The tests included several runs of the performance tests (included with OFED), Intel MPI PingPong benchmark on OpenMPI, krping for FRWRs. Mellanox has been kind enough to test the backported version of the driver internally on their hardware using a VMware provided ESX build. I have also applied and tested this with Doug's k.o/for-4.9 branch (commit 5603910b). Note, that this patch series should be applied all together. I split out the commits so that it may be easier to review. PVRDMA Resources ================ [1] OFA Workshop Presentation - https://openfabrics.org/images/eventpresos/2016presentations/102parardma.pdf [2] Libpvrdma User-level library - http://git.openfabrics.org/?p=~aditr/libpvrdma.git;a=summary Reviewed-by: Jorgen Hansen <jhansen@vmware.com> Reviewed-by: George Zhang <georgezhang@vmware.com> Reviewed-by: Aditya Sarwade <asarwade@vmware.com> Reviewed-by: Bryan Tan <bryantan@vmware.com> Reviewed-by: Leon Romanovsky <leonro@mellanox.com> Signed-off-by: Adit Ranadive <aditr@vmware.com> Signed-off-by: Doug Ledford <dledford@redhat.com>
305 lines
8 KiB
C
305 lines
8 KiB
C
/*
|
|
* Copyright (c) 2012-2016 VMware, Inc. All rights reserved.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of EITHER the GNU General Public License
|
|
* version 2 as published by the Free Software Foundation or the BSD
|
|
* 2-Clause License. This program is distributed in the hope that it
|
|
* will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
|
|
* WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
|
|
* See the GNU General Public License version 2 for more details at
|
|
* http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program available in the file COPYING in the main
|
|
* directory of this source tree.
|
|
*
|
|
* The BSD 2-Clause License
|
|
*
|
|
* Redistribution and use in source and binary forms, with or
|
|
* without modification, are permitted provided that the following
|
|
* conditions are met:
|
|
*
|
|
* - Redistributions of source code must retain the above
|
|
* copyright notice, this list of conditions and the following
|
|
* disclaimer.
|
|
*
|
|
* - Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following
|
|
* disclaimer in the documentation and/or other materials
|
|
* provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
|
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
|
* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
|
|
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
|
|
* OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <linux/errno.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/bitmap.h>
|
|
|
|
#include "pvrdma.h"
|
|
|
|
int pvrdma_page_dir_init(struct pvrdma_dev *dev, struct pvrdma_page_dir *pdir,
|
|
u64 npages, bool alloc_pages)
|
|
{
|
|
u64 i;
|
|
|
|
if (npages > PVRDMA_PAGE_DIR_MAX_PAGES)
|
|
return -EINVAL;
|
|
|
|
memset(pdir, 0, sizeof(*pdir));
|
|
|
|
pdir->dir = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
|
|
&pdir->dir_dma, GFP_KERNEL);
|
|
if (!pdir->dir)
|
|
goto err;
|
|
|
|
pdir->ntables = PVRDMA_PAGE_DIR_TABLE(npages - 1) + 1;
|
|
pdir->tables = kcalloc(pdir->ntables, sizeof(*pdir->tables),
|
|
GFP_KERNEL);
|
|
if (!pdir->tables)
|
|
goto err;
|
|
|
|
for (i = 0; i < pdir->ntables; i++) {
|
|
pdir->tables[i] = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
|
|
(dma_addr_t *)&pdir->dir[i],
|
|
GFP_KERNEL);
|
|
if (!pdir->tables[i])
|
|
goto err;
|
|
}
|
|
|
|
pdir->npages = npages;
|
|
|
|
if (alloc_pages) {
|
|
pdir->pages = kcalloc(npages, sizeof(*pdir->pages),
|
|
GFP_KERNEL);
|
|
if (!pdir->pages)
|
|
goto err;
|
|
|
|
for (i = 0; i < pdir->npages; i++) {
|
|
dma_addr_t page_dma;
|
|
|
|
pdir->pages[i] = dma_alloc_coherent(&dev->pdev->dev,
|
|
PAGE_SIZE,
|
|
&page_dma,
|
|
GFP_KERNEL);
|
|
if (!pdir->pages[i])
|
|
goto err;
|
|
|
|
pvrdma_page_dir_insert_dma(pdir, i, page_dma);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
|
|
err:
|
|
pvrdma_page_dir_cleanup(dev, pdir);
|
|
|
|
return -ENOMEM;
|
|
}
|
|
|
|
static u64 *pvrdma_page_dir_table(struct pvrdma_page_dir *pdir, u64 idx)
|
|
{
|
|
return pdir->tables[PVRDMA_PAGE_DIR_TABLE(idx)];
|
|
}
|
|
|
|
dma_addr_t pvrdma_page_dir_get_dma(struct pvrdma_page_dir *pdir, u64 idx)
|
|
{
|
|
return pvrdma_page_dir_table(pdir, idx)[PVRDMA_PAGE_DIR_PAGE(idx)];
|
|
}
|
|
|
|
static void pvrdma_page_dir_cleanup_pages(struct pvrdma_dev *dev,
|
|
struct pvrdma_page_dir *pdir)
|
|
{
|
|
if (pdir->pages) {
|
|
u64 i;
|
|
|
|
for (i = 0; i < pdir->npages && pdir->pages[i]; i++) {
|
|
dma_addr_t page_dma = pvrdma_page_dir_get_dma(pdir, i);
|
|
|
|
dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
|
|
pdir->pages[i], page_dma);
|
|
}
|
|
|
|
kfree(pdir->pages);
|
|
}
|
|
}
|
|
|
|
static void pvrdma_page_dir_cleanup_tables(struct pvrdma_dev *dev,
|
|
struct pvrdma_page_dir *pdir)
|
|
{
|
|
if (pdir->tables) {
|
|
int i;
|
|
|
|
pvrdma_page_dir_cleanup_pages(dev, pdir);
|
|
|
|
for (i = 0; i < pdir->ntables; i++) {
|
|
u64 *table = pdir->tables[i];
|
|
|
|
if (table)
|
|
dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
|
|
table, pdir->dir[i]);
|
|
}
|
|
|
|
kfree(pdir->tables);
|
|
}
|
|
}
|
|
|
|
void pvrdma_page_dir_cleanup(struct pvrdma_dev *dev,
|
|
struct pvrdma_page_dir *pdir)
|
|
{
|
|
if (pdir->dir) {
|
|
pvrdma_page_dir_cleanup_tables(dev, pdir);
|
|
dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
|
|
pdir->dir, pdir->dir_dma);
|
|
}
|
|
}
|
|
|
|
int pvrdma_page_dir_insert_dma(struct pvrdma_page_dir *pdir, u64 idx,
|
|
dma_addr_t daddr)
|
|
{
|
|
u64 *table;
|
|
|
|
if (idx >= pdir->npages)
|
|
return -EINVAL;
|
|
|
|
table = pvrdma_page_dir_table(pdir, idx);
|
|
table[PVRDMA_PAGE_DIR_PAGE(idx)] = daddr;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int pvrdma_page_dir_insert_umem(struct pvrdma_page_dir *pdir,
|
|
struct ib_umem *umem, u64 offset)
|
|
{
|
|
u64 i = offset;
|
|
int j, entry;
|
|
int ret = 0, len = 0;
|
|
struct scatterlist *sg;
|
|
|
|
if (offset >= pdir->npages)
|
|
return -EINVAL;
|
|
|
|
for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
|
|
len = sg_dma_len(sg) >> PAGE_SHIFT;
|
|
for (j = 0; j < len; j++) {
|
|
dma_addr_t addr = sg_dma_address(sg) +
|
|
umem->page_size * j;
|
|
|
|
ret = pvrdma_page_dir_insert_dma(pdir, i, addr);
|
|
if (ret)
|
|
goto exit;
|
|
|
|
i++;
|
|
}
|
|
}
|
|
|
|
exit:
|
|
return ret;
|
|
}
|
|
|
|
int pvrdma_page_dir_insert_page_list(struct pvrdma_page_dir *pdir,
|
|
u64 *page_list,
|
|
int num_pages)
|
|
{
|
|
int i;
|
|
int ret;
|
|
|
|
if (num_pages > pdir->npages)
|
|
return -EINVAL;
|
|
|
|
for (i = 0; i < num_pages; i++) {
|
|
ret = pvrdma_page_dir_insert_dma(pdir, i, page_list[i]);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void pvrdma_qp_cap_to_ib(struct ib_qp_cap *dst, const struct pvrdma_qp_cap *src)
|
|
{
|
|
dst->max_send_wr = src->max_send_wr;
|
|
dst->max_recv_wr = src->max_recv_wr;
|
|
dst->max_send_sge = src->max_send_sge;
|
|
dst->max_recv_sge = src->max_recv_sge;
|
|
dst->max_inline_data = src->max_inline_data;
|
|
}
|
|
|
|
void ib_qp_cap_to_pvrdma(struct pvrdma_qp_cap *dst, const struct ib_qp_cap *src)
|
|
{
|
|
dst->max_send_wr = src->max_send_wr;
|
|
dst->max_recv_wr = src->max_recv_wr;
|
|
dst->max_send_sge = src->max_send_sge;
|
|
dst->max_recv_sge = src->max_recv_sge;
|
|
dst->max_inline_data = src->max_inline_data;
|
|
}
|
|
|
|
void pvrdma_gid_to_ib(union ib_gid *dst, const union pvrdma_gid *src)
|
|
{
|
|
BUILD_BUG_ON(sizeof(union pvrdma_gid) != sizeof(union ib_gid));
|
|
memcpy(dst, src, sizeof(*src));
|
|
}
|
|
|
|
void ib_gid_to_pvrdma(union pvrdma_gid *dst, const union ib_gid *src)
|
|
{
|
|
BUILD_BUG_ON(sizeof(union pvrdma_gid) != sizeof(union ib_gid));
|
|
memcpy(dst, src, sizeof(*src));
|
|
}
|
|
|
|
void pvrdma_global_route_to_ib(struct ib_global_route *dst,
|
|
const struct pvrdma_global_route *src)
|
|
{
|
|
pvrdma_gid_to_ib(&dst->dgid, &src->dgid);
|
|
dst->flow_label = src->flow_label;
|
|
dst->sgid_index = src->sgid_index;
|
|
dst->hop_limit = src->hop_limit;
|
|
dst->traffic_class = src->traffic_class;
|
|
}
|
|
|
|
void ib_global_route_to_pvrdma(struct pvrdma_global_route *dst,
|
|
const struct ib_global_route *src)
|
|
{
|
|
ib_gid_to_pvrdma(&dst->dgid, &src->dgid);
|
|
dst->flow_label = src->flow_label;
|
|
dst->sgid_index = src->sgid_index;
|
|
dst->hop_limit = src->hop_limit;
|
|
dst->traffic_class = src->traffic_class;
|
|
}
|
|
|
|
void pvrdma_ah_attr_to_ib(struct ib_ah_attr *dst,
|
|
const struct pvrdma_ah_attr *src)
|
|
{
|
|
pvrdma_global_route_to_ib(&dst->grh, &src->grh);
|
|
dst->dlid = src->dlid;
|
|
dst->sl = src->sl;
|
|
dst->src_path_bits = src->src_path_bits;
|
|
dst->static_rate = src->static_rate;
|
|
dst->ah_flags = src->ah_flags;
|
|
dst->port_num = src->port_num;
|
|
memcpy(&dst->dmac, &src->dmac, sizeof(dst->dmac));
|
|
}
|
|
|
|
void ib_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
|
|
const struct ib_ah_attr *src)
|
|
{
|
|
ib_global_route_to_pvrdma(&dst->grh, &src->grh);
|
|
dst->dlid = src->dlid;
|
|
dst->sl = src->sl;
|
|
dst->src_path_bits = src->src_path_bits;
|
|
dst->static_rate = src->static_rate;
|
|
dst->ah_flags = src->ah_flags;
|
|
dst->port_num = src->port_num;
|
|
memcpy(&dst->dmac, &src->dmac, sizeof(dst->dmac));
|
|
}
|