RDMA/core: Add trace points to follow MR allocation
Track the lifetime of ib_mr objects. Here's sample output from a test run
with NFS/RDMA:
<...>-361 [009] 79238.772782: mr_alloc: pd.id=3 mr.id=11 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772812: mr_alloc: pd.id=3 mr.id=12 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772839: mr_alloc: pd.id=3 mr.id=13 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772866: mr_alloc: pd.id=3 mr.id=14 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772893: mr_alloc: pd.id=3 mr.id=15 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772921: mr_alloc: pd.id=3 mr.id=16 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772947: mr_alloc: pd.id=3 mr.id=17 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.772974: mr_alloc: pd.id=3 mr.id=18 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.773001: mr_alloc: pd.id=3 mr.id=19 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.773028: mr_alloc: pd.id=3 mr.id=20 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79238.773055: mr_alloc: pd.id=3 mr.id=21 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.270942: mr_alloc: pd.id=3 mr.id=22 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.270975: mr_alloc: pd.id=3 mr.id=23 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271007: mr_alloc: pd.id=3 mr.id=24 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271036: mr_alloc: pd.id=3 mr.id=25 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271067: mr_alloc: pd.id=3 mr.id=26 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271095: mr_alloc: pd.id=3 mr.id=27 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271121: mr_alloc: pd.id=3 mr.id=28 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271153: mr_alloc: pd.id=3 mr.id=29 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271181: mr_alloc: pd.id=3 mr.id=30 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271208: mr_alloc: pd.id=3 mr.id=31 type=MEM_REG max_num_sg=30 rc=0
<...>-361 [009] 79240.271236: mr_alloc: pd.id=3 mr.id=32 type=MEM_REG max_num_sg=30 rc=0
<...>-4351 [001] 79242.299400: mr_dereg: mr.id=32
<...>-4351 [001] 79242.299467: mr_dereg: mr.id=31
<...>-4351 [001] 79242.299554: mr_dereg: mr.id=30
<...>-4351 [001] 79242.299615: mr_dereg: mr.id=29
<...>-4351 [001] 79242.299684: mr_dereg: mr.id=28
<...>-4351 [001] 79242.299748: mr_dereg: mr.id=27
<...>-4351 [001] 79242.299812: mr_dereg: mr.id=26
<...>-4351 [001] 79242.299874: mr_dereg: mr.id=25
<...>-4351 [001] 79242.299944: mr_dereg: mr.id=24
<...>-4351 [001] 79242.300009: mr_dereg: mr.id=23
<...>-4351 [001] 79242.300190: mr_dereg: mr.id=22
<...>-4351 [001] 79242.300263: mr_dereg: mr.id=21
<...>-4351 [001] 79242.300326: mr_dereg: mr.id=20
<...>-4351 [001] 79242.300388: mr_dereg: mr.id=19
<...>-4351 [001] 79242.300450: mr_dereg: mr.id=18
<...>-4351 [001] 79242.300516: mr_dereg: mr.id=17
<...>-4351 [001] 79242.300629: mr_dereg: mr.id=16
<...>-4351 [001] 79242.300718: mr_dereg: mr.id=15
<...>-4351 [001] 79242.300784: mr_dereg: mr.id=14
<...>-4351 [001] 79242.300879: mr_dereg: mr.id=13
<...>-4351 [001] 79242.300945: mr_dereg: mr.id=12
<...>-4351 [001] 79242.301012: mr_dereg: mr.id=11
Some features of the output:
- The lifetime and owner PD of each MR is clearly visible.
- The type of MR is captured, as is the SGE array size.
- Failing MR allocation can be recorded.
Link: https://lore.kernel.org/r/20191218201820.30584.34636.stgit@manet.1015granger.net
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
alistair/sensors
Chuck Lever
Jason Gunthorpe
parent
3e5901cbfc
commit
622db5b643
|
|
@ -52,6 +52,7 @@
|
|||
#include <rdma/rw.h>
|
||||
|
||||
#include "core_priv.h"
|
||||
#include <trace/events/rdma_core.h>
|
||||
|
||||
#include <trace/events/rdma_core.h>
|
||||
|
||||
|
|
@ -1999,6 +2000,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
|
|||
struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
|
||||
int ret;
|
||||
|
||||
trace_mr_dereg(mr);
|
||||
rdma_restrack_del(&mr->res);
|
||||
ret = mr->device->ops.dereg_mr(mr, udata);
|
||||
if (!ret) {
|
||||
|
|
@ -2030,11 +2032,16 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
|
|||
{
|
||||
struct ib_mr *mr;
|
||||
|
||||
if (!pd->device->ops.alloc_mr)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
if (!pd->device->ops.alloc_mr) {
|
||||
mr = ERR_PTR(-EOPNOTSUPP);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (WARN_ON_ONCE(mr_type == IB_MR_TYPE_INTEGRITY))
|
||||
return ERR_PTR(-EINVAL);
|
||||
if (mr_type == IB_MR_TYPE_INTEGRITY) {
|
||||
WARN_ON_ONCE(1);
|
||||
mr = ERR_PTR(-EINVAL);
|
||||
goto out;
|
||||
}
|
||||
|
||||
mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);
|
||||
if (!IS_ERR(mr)) {
|
||||
|
|
@ -2050,6 +2057,8 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
|
|||
mr->sig_attrs = NULL;
|
||||
}
|
||||
|
||||
out:
|
||||
trace_mr_alloc(pd, mr_type, max_num_sg, mr);
|
||||
return mr;
|
||||
}
|
||||
EXPORT_SYMBOL(ib_alloc_mr_user);
|
||||
|
|
@ -2074,21 +2083,27 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
|
|||
struct ib_sig_attrs *sig_attrs;
|
||||
|
||||
if (!pd->device->ops.alloc_mr_integrity ||
|
||||
!pd->device->ops.map_mr_sg_pi)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
!pd->device->ops.map_mr_sg_pi) {
|
||||
mr = ERR_PTR(-EOPNOTSUPP);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (!max_num_meta_sg)
|
||||
return ERR_PTR(-EINVAL);
|
||||
if (!max_num_meta_sg) {
|
||||
mr = ERR_PTR(-EINVAL);
|
||||
goto out;
|
||||
}
|
||||
|
||||
sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
|
||||
if (!sig_attrs)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
if (!sig_attrs) {
|
||||
mr = ERR_PTR(-ENOMEM);
|
||||
goto out;
|
||||
}
|
||||
|
||||
mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
|
||||
max_num_meta_sg);
|
||||
if (IS_ERR(mr)) {
|
||||
kfree(sig_attrs);
|
||||
return mr;
|
||||
goto out;
|
||||
}
|
||||
|
||||
mr->device = pd->device;
|
||||
|
|
@ -2102,6 +2117,8 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
|
|||
mr->type = IB_MR_TYPE_INTEGRITY;
|
||||
mr->sig_attrs = sig_attrs;
|
||||
|
||||
out:
|
||||
trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
|
||||
return mr;
|
||||
}
|
||||
EXPORT_SYMBOL(ib_alloc_mr_integrity);
|
||||
|
|
|
|||
|
|
@ -266,6 +266,129 @@ TRACE_EVENT(cq_free,
|
|||
TP_printk("cq.id=%u", __entry->cq_id)
|
||||
);
|
||||
|
||||
/**
|
||||
** Memory Region events
|
||||
**/
|
||||
|
||||
/*
|
||||
* enum ib_mr_type, from include/rdma/ib_verbs.h
|
||||
*/
|
||||
#define IB_MR_TYPE_LIST \
|
||||
ib_mr_type_item(MEM_REG) \
|
||||
ib_mr_type_item(SG_GAPS) \
|
||||
ib_mr_type_item(DM) \
|
||||
ib_mr_type_item(USER) \
|
||||
ib_mr_type_item(DMA) \
|
||||
ib_mr_type_end(INTEGRITY)
|
||||
|
||||
#undef ib_mr_type_item
|
||||
#undef ib_mr_type_end
|
||||
|
||||
#define ib_mr_type_item(x) TRACE_DEFINE_ENUM(IB_MR_TYPE_##x);
|
||||
#define ib_mr_type_end(x) TRACE_DEFINE_ENUM(IB_MR_TYPE_##x);
|
||||
|
||||
IB_MR_TYPE_LIST
|
||||
|
||||
#undef ib_mr_type_item
|
||||
#undef ib_mr_type_end
|
||||
|
||||
#define ib_mr_type_item(x) { IB_MR_TYPE_##x, #x },
|
||||
#define ib_mr_type_end(x) { IB_MR_TYPE_##x, #x }
|
||||
|
||||
#define rdma_show_ib_mr_type(x) \
|
||||
__print_symbolic(x, IB_MR_TYPE_LIST)
|
||||
|
||||
TRACE_EVENT(mr_alloc,
|
||||
TP_PROTO(
|
||||
const struct ib_pd *pd,
|
||||
enum ib_mr_type mr_type,
|
||||
u32 max_num_sg,
|
||||
const struct ib_mr *mr
|
||||
),
|
||||
|
||||
TP_ARGS(pd, mr_type, max_num_sg, mr),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u32, pd_id)
|
||||
__field(u32, mr_id)
|
||||
__field(u32, max_num_sg)
|
||||
__field(int, rc)
|
||||
__field(unsigned long, mr_type)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->pd_id = pd->res.id;
|
||||
if (IS_ERR(mr)) {
|
||||
__entry->mr_id = 0;
|
||||
__entry->rc = PTR_ERR(mr);
|
||||
} else {
|
||||
__entry->mr_id = mr->res.id;
|
||||
__entry->rc = 0;
|
||||
}
|
||||
__entry->max_num_sg = max_num_sg;
|
||||
__entry->mr_type = mr_type;
|
||||
),
|
||||
|
||||
TP_printk("pd.id=%u mr.id=%u type=%s max_num_sg=%u rc=%d",
|
||||
__entry->pd_id, __entry->mr_id,
|
||||
rdma_show_ib_mr_type(__entry->mr_type),
|
||||
__entry->max_num_sg, __entry->rc)
|
||||
);
|
||||
|
||||
TRACE_EVENT(mr_integ_alloc,
|
||||
TP_PROTO(
|
||||
const struct ib_pd *pd,
|
||||
u32 max_num_data_sg,
|
||||
u32 max_num_meta_sg,
|
||||
const struct ib_mr *mr
|
||||
),
|
||||
|
||||
TP_ARGS(pd, max_num_data_sg, max_num_meta_sg, mr),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u32, pd_id)
|
||||
__field(u32, mr_id)
|
||||
__field(u32, max_num_data_sg)
|
||||
__field(u32, max_num_meta_sg)
|
||||
__field(int, rc)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->pd_id = pd->res.id;
|
||||
if (IS_ERR(mr)) {
|
||||
__entry->mr_id = 0;
|
||||
__entry->rc = PTR_ERR(mr);
|
||||
} else {
|
||||
__entry->mr_id = mr->res.id;
|
||||
__entry->rc = 0;
|
||||
}
|
||||
__entry->max_num_data_sg = max_num_data_sg;
|
||||
__entry->max_num_meta_sg = max_num_meta_sg;
|
||||
),
|
||||
|
||||
TP_printk("pd.id=%u mr.id=%u max_num_data_sg=%u max_num_meta_sg=%u rc=%d",
|
||||
__entry->pd_id, __entry->mr_id, __entry->max_num_data_sg,
|
||||
__entry->max_num_meta_sg, __entry->rc)
|
||||
);
|
||||
|
||||
TRACE_EVENT(mr_dereg,
|
||||
TP_PROTO(
|
||||
const struct ib_mr *mr
|
||||
),
|
||||
|
||||
TP_ARGS(mr),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u32, id)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->id = mr->res.id;
|
||||
),
|
||||
|
||||
TP_printk("mr.id=%u", __entry->id)
|
||||
);
|
||||
|
||||
#endif /* _TRACE_RDMA_CORE_H */
|
||||
|
||||
#include <trace/define_trace.h>
|
||||
|
|
|
|||
Loading…
Reference in New Issue