Commit 91a7c58f authored by Leon Romanovsky's avatar Leon Romanovsky Committed by Jason Gunthorpe
Browse files

RDMA: Restore ability to fail on PD deallocate 

The IB verbs objects are counted by the kernel and ib_core ensures that
deallocate PD will success so it will be called once all other objects
that depends on PD will be released. This is achieved by managing various
reference counters on such objects.

The mlx5 driver didn't follow this standard flow when allowed DEVX objects
that are not managed by ib_core to be interleaved with the ones under
ib_core responsibility.

In such interleaved scenarios deallocate command can fail and ib_core will
leave uobject in internal DB and attempt to clean it later to free
resources anyway.

This change partially restores returned value from dealloc_pd() for all
drivers, but keeping in mind that non-DEVX devices and kernel verbs paths
shouldn't fail.

Fixes: 21a428a0 ("RDMA: Handle PD allocations by IB/core")
Link: https://lore.kernel.org/r/20200907120921.476363-2-leon@kernel.org


Signed-off-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 558d52b2

drivers/infiniband/core/uverbs_std_types.c

+1 −2
Original line number Original line Diff line number Diff line
@@ -122,8 +122,7 @@ static int uverbs_free_pd(struct ib_uobject *uobject, 
	if (ret)

	if (ret)

		return ret;

		return ret;





	ib_dealloc_pd_user(pd, &attrs->driver_udata);

	return ib_dealloc_pd_user(pd, &attrs->driver_udata);

	return 0;

}

}





void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue)

void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue)

drivers/infiniband/core/verbs.c

+6 −2
Original line number Original line Diff line number Diff line
@@ -329,7 +329,7 @@ EXPORT_SYMBOL(__ib_alloc_pd); 
 * exist.  The caller is responsible to synchronously destroy them and

 * exist.  The caller is responsible to synchronously destroy them and

 * guarantee no new allocations will happen.

 * guarantee no new allocations will happen.

 */

 */

void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)

int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)

{

{

	int ret;

	int ret;
@@ -343,9 +343,13 @@ void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) 
	   requires the caller to guarantee we can't race here. */

	   requires the caller to guarantee we can't race here. */

	WARN_ON(atomic_read(&pd->usecnt));

	WARN_ON(atomic_read(&pd->usecnt));





	ret = pd->device->ops.dealloc_pd(pd, udata);

	if (ret)

		return ret;



	rdma_restrack_del(&pd->res);

	rdma_restrack_del(&pd->res);

	pd->device->ops.dealloc_pd(pd, udata);

	kfree(pd);

	kfree(pd);

	return ret;

}

}

EXPORT_SYMBOL(ib_dealloc_pd_user);

EXPORT_SYMBOL(ib_dealloc_pd_user);

drivers/infiniband/hw/bnxt_re/ib_verbs.c

+2 −1
Original line number Original line Diff line number Diff line
@@ -532,7 +532,7 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) 
}

}





/* Protection Domains */

/* Protection Domains */

void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)

int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)

{

{

	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);

	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);

	struct bnxt_re_dev *rdev = pd->rdev;

	struct bnxt_re_dev *rdev = pd->rdev;
@@ -542,6 +542,7 @@ void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) 
	if (pd->qplib_pd.id)

	if (pd->qplib_pd.id)

		bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,

		bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,

				      &pd->qplib_pd);

				      &pd->qplib_pd);

	return 0;

}

}





int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)

int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)

drivers/infiniband/hw/bnxt_re/ib_verbs.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -163,7 +163,7 @@ int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, 
enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,

enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,

					    u8 port_num);

					    u8 port_num);

int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);

int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);

void bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);

int bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);

int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,

int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,

		      struct ib_udata *udata);

		      struct ib_udata *udata);

int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);

int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);

drivers/infiniband/hw/cxgb4/provider.c

+2 −1
Original line number Original line Diff line number Diff line
@@ -190,7 +190,7 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 
	return ret;

	return ret;

}

}





static void c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)

static int c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)

{

{

	struct c4iw_dev *rhp;

	struct c4iw_dev *rhp;

	struct c4iw_pd *php;

	struct c4iw_pd *php;
@@ -202,6 +202,7 @@ static void c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata) 
	mutex_lock(&rhp->rdev.stats.lock);

	mutex_lock(&rhp->rdev.stats.lock);

	rhp->rdev.stats.pd.cur--;

	rhp->rdev.stats.pd.cur--;

	mutex_unlock(&rhp->rdev.stats.lock);

	mutex_unlock(&rhp->rdev.stats.lock);

	return 0;

}

}





static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)

static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)