From 143e8cc030a8be486937485f01e0cc75f338612e Mon Sep 17 00:00:00 2001
From: Sean Hefty <sean.hefty@intel.com>
Date: Thu, 19 Jul 2012 14:14:43 -0700
Subject: [PATCH] Refresh of cma-rm-pd

---
 src/cma.c | 50 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/src/cma.c b/src/cma.c
index bad8bce3..a190deec 100755
--- a/src/cma.c
+++ b/src/cma.c
@@ -76,6 +76,7 @@ struct cma_device {
 	struct ibv_pd	   *pd;
 	uint64_t	    guid;
 	int		    port_cnt;
+	int		    refcnt;
 	int		    max_qpsize;
 	uint8_t		    max_initiator_depth;
 	uint8_t		    max_responder_resources;
@@ -122,6 +123,7 @@ static int cma_dev_cnt;
 static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
 static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
 int af_ib_support;
+static int verbs_refcnt;
 
 #define container_of(ptr, type, field) \
 	((type *) ((void *)ptr - offsetof(type, field)))
@@ -129,10 +131,12 @@ int af_ib_support;
 static void ucma_cleanup(void)
 {
 	ucma_ib_cleanup();
+	ucma_release_verbs();
 
 	if (cma_dev_cnt) {
 		while (cma_dev_cnt--) {
-			ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
+			if (cma_dev_array[cma_dev_cnt].refcnt)
+				ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
 			ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
 		}
 
@@ -235,7 +239,7 @@ int ucma_init(void)
 		goto err2;
 	}
 		
-	cma_dev_array = malloc(sizeof *cma_dev * dev_cnt);
+	cma_dev_array = calloc(dev_cnt, sizeof *cma_dev);
 	if (!cma_dev_array) {
 		ret = ERR(ENOMEM);
 		goto err2;
@@ -252,13 +256,6 @@ int ucma_init(void)
 			goto err3;
 		}
 
-		cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
-		if (!cma_dev->pd) {
-			ibv_close_device(cma_dev->verbs);
-			ret = ERR(ENOMEM);
-			goto err3;
-		}
-
 		i++;
 		ret = ibv_query_device(cma_dev->verbs, &attr);
 		if (ret) {
@@ -280,10 +277,8 @@ int ucma_init(void)
 	return 0;
 
 err3:
-	while (i--) {
-		ibv_dealloc_pd(cma_dev_array[i].pd);
+	while (i--)
 		ibv_close_device(cma_dev_array[i].verbs);
-	}
 	free(cma_dev_array);
 err2:
 	ibv_free_device_list(dev_list);
@@ -358,15 +353,34 @@ static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
 
 	for (i = 0; i < cma_dev_cnt; i++) {
 		cma_dev = &cma_dev_array[i];
-		if (cma_dev->guid == guid) {
-			id_priv->cma_dev = cma_dev;
-			id_priv->id.verbs = cma_dev->verbs;
-			id_priv->id.pd = cma_dev->pd;
-			return 0;
-		}
+		if (cma_dev->guid == guid)
+			goto match;
 	}
 
 	return ERR(ENODEV);
+match:
+	pthread_mutex_lock(&mut);
+	if (!cma_dev->refcnt++) {
+		cma_dev->pd = ibv_alloc_pd(cma_dev_array[i].verbs);
+		if (!cma_dev->pd) {
+			cma_dev->refcnt--;
+			pthread_mutex_unlock(&mut);
+			return ERR(ENOMEM);
+		}
+	}
+	pthread_mutex_unlock(&mut);
+	id_priv->cma_dev = cma_dev;
+	id_priv->id.verbs = cma_dev->verbs;
+	id_priv->id.pd = cma_dev->pd;
+	return 0;
+}
+
+static void ucma_put_device(struct cma_device *cma_dev)
+{
+	pthread_mutex_lock(&mut);
+	if (!--cma_dev->verbs_refcnt)
+		ibv_dealloc_pd(cma_dev_array[i].pd)
+	pthread_mutex_unlock(&mut);
 }
 
 static void ucma_free_id(struct cma_id_private *id_priv)
-- 
2.45.2