From 6972e479f8b886ca4577de6c4ca30e1f9cbc660d Mon Sep 17 00:00:00 2001
From: Sean Hefty <sean.hefty@intel.com>
Date: Thu, 19 Jul 2012 10:09:48 -0700
Subject: [PATCH] librdmacm: Only allocate verbs resources when needed

The librdmacm allocates a PD per device on initialization.  Although
we need to maintain the device list while the library is loaded
(see rdma_get_devices), we can reduce the overhead by only allocating
verbs resources when they are needed.

This allows the rsocket preload library to support fork for
applications that spawn connections off to child processes.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
---
 src/cma.c | 54 +++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 35 insertions(+), 19 deletions(-)

diff --git a/src/cma.c b/src/cma.c
index bad8bce3..65b52bf7 100755
--- a/src/cma.c
+++ b/src/cma.c
@@ -76,6 +76,7 @@ struct cma_device {
 	struct ibv_pd	   *pd;
 	uint64_t	    guid;
 	int		    port_cnt;
+	int		    refcnt;
 	int		    max_qpsize;
 	uint8_t		    max_initiator_depth;
 	uint8_t		    max_responder_resources;
@@ -122,6 +123,7 @@ static int cma_dev_cnt;
 static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
 static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
 int af_ib_support;
+static int verbs_refcnt;
 
 #define container_of(ptr, type, field) \
 	((type *) ((void *)ptr - offsetof(type, field)))
@@ -132,7 +134,8 @@ static void ucma_cleanup(void)
 
 	if (cma_dev_cnt) {
 		while (cma_dev_cnt--) {
-			ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
+			if (cma_dev_array[cma_dev_cnt].refcnt)
+				ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
 			ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
 		}
 
@@ -235,7 +238,7 @@ int ucma_init(void)
 		goto err2;
 	}
 		
-	cma_dev_array = malloc(sizeof *cma_dev * dev_cnt);
+	cma_dev_array = calloc(dev_cnt, sizeof *cma_dev);
 	if (!cma_dev_array) {
 		ret = ERR(ENOMEM);
 		goto err2;
@@ -252,13 +255,6 @@ int ucma_init(void)
 			goto err3;
 		}
 
-		cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
-		if (!cma_dev->pd) {
-			ibv_close_device(cma_dev->verbs);
-			ret = ERR(ENOMEM);
-			goto err3;
-		}
-
 		i++;
 		ret = ibv_query_device(cma_dev->verbs, &attr);
 		if (ret) {
@@ -280,10 +276,8 @@ int ucma_init(void)
 	return 0;
 
 err3:
-	while (i--) {
-		ibv_dealloc_pd(cma_dev_array[i].pd);
+	while (i--)
 		ibv_close_device(cma_dev_array[i].verbs);
-	}
 	free(cma_dev_array);
 err2:
 	ibv_free_device_list(dev_list);
@@ -354,23 +348,45 @@ void rdma_destroy_event_channel(struct rdma_event_channel *channel)
 static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
 {
 	struct cma_device *cma_dev;
-	int i;
+	int i, ret = 0;
 
 	for (i = 0; i < cma_dev_cnt; i++) {
 		cma_dev = &cma_dev_array[i];
-		if (cma_dev->guid == guid) {
-			id_priv->cma_dev = cma_dev;
-			id_priv->id.verbs = cma_dev->verbs;
-			id_priv->id.pd = cma_dev->pd;
-			return 0;
-		}
+		if (cma_dev->guid == guid)
+			goto match;
 	}
 
 	return ERR(ENODEV);
+match:
+	pthread_mutex_lock(&mut);
+	if (!cma_dev->refcnt++) {
+		cma_dev->pd = ibv_alloc_pd(cma_dev_array[i].verbs);
+		if (!cma_dev->pd) {
+			cma_dev->refcnt--;
+			ret = ERR(ENOMEM);
+			goto out;
+		}
+	}
+	id_priv->cma_dev = cma_dev;
+	id_priv->id.verbs = cma_dev->verbs;
+	id_priv->id.pd = cma_dev->pd;
+out:
+	pthread_mutex_unlock(&mut);
+	return ret;
+}
+
+static void ucma_put_device(struct cma_device *cma_dev)
+{
+	pthread_mutex_lock(&mut);
+	if (!--cma_dev->refcnt)
+		ibv_dealloc_pd(cma_dev->pd);
+	pthread_mutex_unlock(&mut);
 }
 
 static void ucma_free_id(struct cma_id_private *id_priv)
 {
+	if (id_priv->cma_dev)
+		ucma_put_device(id_priv->cma_dev);
 	pthread_cond_destroy(&id_priv->cond);
 	pthread_mutex_destroy(&id_priv->mut);
 	if (id_priv->id.route.path_rec)
-- 
2.45.2