![Thomas Richter](/assets/img/avatar_default.png)
Connection creation with SMC-R starts through an internal TCP-connection. The Ethernet interface for this TCP-connection is not restricted to the Ethernet interface of a RoCE device. Any existing Ethernet interface belonging to the same physical net can be used, as long as there is a defined relation between the Ethernet interface and some RoCE devices. This relation is defined with the help of an identification string called "Physical Net ID" or short "pnet ID". Information about defined pnet IDs and their related Ethernet interfaces and RoCE devices is stored in the SMC-R pnet table. A pnet table entry consists of the identifying pnet ID and the associated network and IB device. This patch adds pnet table configuration support using the generic netlink message interface referring to network and IB device by their names. Commands exist to add, delete, and display pnet table entries, and to flush or display the entire pnet table. There are cross-checks to verify whether the ethernet interfaces or infiniband devices really exist in the system. If either device is not available, the pnet ID entry is not created. Loss of network devices and IB devices is also monitored; a pnet ID entry is removed when an associated network or IB device is removed. Signed-off-by: Thomas Richter <tmricht@linux.vnet.ibm.com> Signed-off-by: Ursula Braun <ubraun@linux.vnet.ibm.com> Signed-off-by: David S. Miller <davem@davemloft.net>
146 lines
3.9 KiB
C
146 lines
3.9 KiB
C
/*
|
|
* Shared Memory Communications over RDMA (SMC-R) and RoCE
|
|
*
|
|
* IB infrastructure:
|
|
* Establish SMC-R as an Infiniband Client to be notified about added and
|
|
* removed IB devices of type RDMA.
|
|
* Determine device and port characteristics for these IB devices.
|
|
*
|
|
* Copyright IBM Corp. 2016
|
|
*
|
|
* Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
|
|
*/
|
|
|
|
#include <linux/random.h>
|
|
#include <rdma/ib_verbs.h>
|
|
|
|
#include "smc_pnet.h"
|
|
#include "smc_ib.h"
|
|
#include "smc.h"
|
|
|
|
struct smc_ib_devices smc_ib_devices = { /* smc-registered ib devices */
|
|
.lock = __SPIN_LOCK_UNLOCKED(smc_ib_devices.lock),
|
|
.list = LIST_HEAD_INIT(smc_ib_devices.list),
|
|
};
|
|
|
|
#define SMC_LOCAL_SYSTEMID_RESET "%%%%%%%"
|
|
|
|
u8 local_systemid[SMC_SYSTEMID_LEN] = SMC_LOCAL_SYSTEMID_RESET; /* unique system
|
|
* identifier
|
|
*/
|
|
|
|
static int smc_ib_fill_gid_and_mac(struct smc_ib_device *smcibdev, u8 ibport)
|
|
{
|
|
struct net_device *ndev;
|
|
int rc;
|
|
|
|
rc = ib_query_gid(smcibdev->ibdev, ibport, 0,
|
|
&smcibdev->gid[ibport - 1], NULL);
|
|
/* the SMC protocol requires specification of the roce MAC address;
|
|
* if net_device cannot be determined, it can be derived from gid 0
|
|
*/
|
|
ndev = smcibdev->ibdev->get_netdev(smcibdev->ibdev, ibport);
|
|
if (ndev) {
|
|
memcpy(&smcibdev->mac, ndev->dev_addr, ETH_ALEN);
|
|
} else if (!rc) {
|
|
memcpy(&smcibdev->mac[ibport - 1][0],
|
|
&smcibdev->gid[ibport - 1].raw[8], 3);
|
|
memcpy(&smcibdev->mac[ibport - 1][3],
|
|
&smcibdev->gid[ibport - 1].raw[13], 3);
|
|
smcibdev->mac[ibport - 1][0] &= ~0x02;
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
/* Create an identifier unique for this instance of SMC-R.
|
|
* The MAC-address of the first active registered IB device
|
|
* plus a random 2-byte number is used to create this identifier.
|
|
* This name is delivered to the peer during connection initialization.
|
|
*/
|
|
static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev,
|
|
u8 ibport)
|
|
{
|
|
memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1],
|
|
sizeof(smcibdev->mac[ibport - 1]));
|
|
get_random_bytes(&local_systemid[0], 2);
|
|
}
|
|
|
|
bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport)
|
|
{
|
|
return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE;
|
|
}
|
|
|
|
int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
|
|
{
|
|
int rc;
|
|
|
|
memset(&smcibdev->pattr[ibport - 1], 0,
|
|
sizeof(smcibdev->pattr[ibport - 1]));
|
|
rc = ib_query_port(smcibdev->ibdev, ibport,
|
|
&smcibdev->pattr[ibport - 1]);
|
|
if (rc)
|
|
goto out;
|
|
rc = smc_ib_fill_gid_and_mac(smcibdev, ibport);
|
|
if (rc)
|
|
goto out;
|
|
if (!strncmp(local_systemid, SMC_LOCAL_SYSTEMID_RESET,
|
|
sizeof(local_systemid)) &&
|
|
smc_ib_port_active(smcibdev, ibport))
|
|
/* create unique system identifier */
|
|
smc_ib_define_local_systemid(smcibdev, ibport);
|
|
out:
|
|
return rc;
|
|
}
|
|
|
|
static struct ib_client smc_ib_client;
|
|
|
|
/* callback function for ib_register_client() */
|
|
static void smc_ib_add_dev(struct ib_device *ibdev)
|
|
{
|
|
struct smc_ib_device *smcibdev;
|
|
|
|
if (ibdev->node_type != RDMA_NODE_IB_CA)
|
|
return;
|
|
|
|
smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL);
|
|
if (!smcibdev)
|
|
return;
|
|
|
|
smcibdev->ibdev = ibdev;
|
|
|
|
spin_lock(&smc_ib_devices.lock);
|
|
list_add_tail(&smcibdev->list, &smc_ib_devices.list);
|
|
spin_unlock(&smc_ib_devices.lock);
|
|
ib_set_client_data(ibdev, &smc_ib_client, smcibdev);
|
|
}
|
|
|
|
/* callback function for ib_register_client() */
|
|
static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data)
|
|
{
|
|
struct smc_ib_device *smcibdev;
|
|
|
|
smcibdev = ib_get_client_data(ibdev, &smc_ib_client);
|
|
ib_set_client_data(ibdev, &smc_ib_client, NULL);
|
|
spin_lock(&smc_ib_devices.lock);
|
|
list_del_init(&smcibdev->list); /* remove from smc_ib_devices */
|
|
spin_unlock(&smc_ib_devices.lock);
|
|
smc_pnet_remove_by_ibdev(smcibdev);
|
|
kfree(smcibdev);
|
|
}
|
|
|
|
static struct ib_client smc_ib_client = {
|
|
.name = "smc_ib",
|
|
.add = smc_ib_add_dev,
|
|
.remove = smc_ib_remove_dev,
|
|
};
|
|
|
|
int __init smc_ib_register_client(void)
|
|
{
|
|
return ib_register_client(&smc_ib_client);
|
|
}
|
|
|
|
void smc_ib_unregister_client(void)
|
|
{
|
|
ib_unregister_client(&smc_ib_client);
|
|
}
|