redonkable/remarkable-linux
redonkable/remarkable-linux
1
0
Fork 0
Code Releases Activity

IB/rdmavt: Use per-CPU reference count for MRs

Browse source 
Having per-CPU reference count for each MR prevents
cache-line bouncing across the system. Thus, it
prevents bottlenecks. Use per-CPU reference counts
per MR.

The per-CPU reference count for FMRs is used in
atomic mode to allow accurate testing of the busy
state. Other MR types run in per-CPU mode MR until
they're freed.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Sebastian Sanchez <sebastian.sanchez@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
This commit is contained in:
Sebastian Sanchez 2017-02-08 05:26:31 -08:00 committed by Doug Ledford
parent f3e862cb68
commit 338adfdddf
2 changed files with 43 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
drivers/infiniband/sw/rdmavt/mr.c
View file

@ -120,10 +120,19 @@ static void rvt_deinit_mregion(struct rvt_mregion *mr)
mr->mapsz = 0;
while (i)
kfree(mr->map[--i]);
percpu_ref_exit(&mr->refcount);
}
static void __rvt_mregion_complete(struct percpu_ref *ref)
{
struct rvt_mregion *mr = container_of(ref, struct rvt_mregion,
refcount);
complete(&mr->comp);
}
static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
int count)
int count, unsigned int percpu_flags)
{
int m, i = 0;
struct rvt_dev_info *dev = ib_to_rvt(pd->device);
@ -133,19 +142,23 @@ static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
for (; i < m; i++) {
mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL,
dev->dparms.node);
if (!mr->map[i]) {
rvt_deinit_mregion(mr);
return -ENOMEM;
}
if (!mr->map[i])
goto bail;
mr->mapsz++;
}
init_completion(&mr->comp);
/* count returning the ptr to user */
atomic_set(&mr->refcount, 1);
if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete,
percpu_flags, GFP_KERNEL))
goto bail;
atomic_set(&mr->lkey_invalid, 0);
mr->pd = pd;
mr->max_segs = count;
return 0;
bail:
rvt_deinit_mregion(mr);
return -ENOMEM;
}
/**
@ -180,8 +193,7 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
if (!tmr) {
rcu_assign_pointer(dev->dma_mr, mr);
mr->lkey_published = 1;
} else {
rvt_put_mr(mr);
rvt_get_mr(mr);
}
goto success;
}
@ -239,11 +251,14 @@ static void rvt_free_lkey(struct rvt_mregion *mr)
int freed = 0;
spin_lock_irqsave(&rkt->lock, flags);
if (!mr->lkey_published)
goto out;
if (lkey == 0) {
RCU_INIT_POINTER(dev->dma_mr, NULL);
if (!lkey) {
if (mr->lkey_published) {
RCU_INIT_POINTER(dev->dma_mr, NULL);
rvt_put_mr(mr);
}
} else {
if (!mr->lkey_published)
goto out;
r = lkey >> (32 - dev->dparms.lkey_table_size);
RCU_INIT_POINTER(rkt->table[r], NULL);
}
@ -253,7 +268,7 @@ out:
spin_unlock_irqrestore(&rkt->lock, flags);
if (freed) {
synchronize_rcu();
rvt_put_mr(mr);
percpu_ref_kill(&mr->refcount);
}
}
@ -269,7 +284,7 @@ static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
if (!mr)
goto bail;
rval = rvt_init_mregion(&mr->mr, pd, count);
rval = rvt_init_mregion(&mr->mr, pd, count, 0);
if (rval)
goto bail;
/*
@ -294,8 +309,8 @@ bail:
static void __rvt_free_mr(struct rvt_mr *mr)
{
rvt_deinit_mregion(&mr->mr);
rvt_free_lkey(&mr->mr);
rvt_deinit_mregion(&mr->mr);
kfree(mr);
}
@ -323,7 +338,7 @@ struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc)
goto bail;
}
rval = rvt_init_mregion(&mr->mr, pd, 0);
rval = rvt_init_mregion(&mr->mr, pd, 0, 0);
if (rval) {
ret = ERR_PTR(rval);
goto bail;
@ -445,8 +460,8 @@ int rvt_dereg_mr(struct ib_mr *ibmr)
timeout = wait_for_completion_timeout(&mr->mr.comp, 5 * HZ);
if (!timeout) {
rvt_pr_err(rdi,
"rvt_dereg_mr timeout mr %p pd %p refcount %u\n",
mr, mr->mr.pd, atomic_read(&mr->mr.refcount));
"rvt_dereg_mr timeout mr %p pd %p\n",
mr, mr->mr.pd);
rvt_get_mr(&mr->mr);
ret = -EBUSY;
goto out;
@ -623,7 +638,8 @@ struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
if (!fmr)
goto bail;
rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages);
rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages,
PERCPU_REF_INIT_ATOMIC);
if (rval)
goto bail;
@ -674,11 +690,12 @@ int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
struct rvt_fmr *fmr = to_ifmr(ibfmr);
struct rvt_lkey_table *rkt;
unsigned long flags;
int m, n, i;
int m, n;
unsigned long i;
u32 ps;
struct rvt_dev_info *rdi = ib_to_rvt(ibfmr->device);
i = atomic_read(&fmr->mr.refcount);
i = atomic_long_read(&fmr->mr.refcount.count);
if (i > 2)
return -EBUSY;

10
include/rdma/rdmavt_mr.h
View file

@ -52,6 +52,7 @@
* For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
* drivers no longer need access to the MR directly.
*/
#include <linux/percpu-refcount.h>
/*
* A segment is a linear region of low physical memory.
@ -79,11 +80,11 @@ struct rvt_mregion {
int access_flags;
u32 max_segs; /* number of rvt_segs in all the arrays */
u32 mapsz; /* size of the map array */
atomic_t lkey_invalid; /* true if current lkey is invalid */
u8 page_shift; /* 0 - non unform/non powerof2 sizes */
u8 lkey_published; /* in global table */
atomic_t lkey_invalid; /* true if current lkey is invalid */
struct percpu_ref refcount;
struct completion comp; /* complete when refcount goes to zero */
atomic_t refcount;
struct rvt_segarray *map[0]; /* the segments */
};
@ -123,13 +124,12 @@ struct rvt_sge_state {
static inline void rvt_put_mr(struct rvt_mregion *mr)
{
if (unlikely(atomic_dec_and_test(&mr->refcount)))
complete(&mr->comp);
percpu_ref_put(&mr->refcount);
}
static inline void rvt_get_mr(struct rvt_mregion *mr)
{
atomic_inc(&mr->refcount);
percpu_ref_get(&mr->refcount);
}
static inline void rvt_put_ss(struct rvt_sge_state *ss)