Refresh of cma-rm-pd

diff --git a/src/cma.c b/src/cma.c
index bad8bce32e0c91590a0c1b921bc199c4eb81c97b..a190deecf82a2b565f8fc4e4d8ed03c3354f1ce2 100755 (executable)
--- a/src/cma.c
+++ b/src/cma.c
@@ -76,6 +76,7 @@ struct cma_device {
        struct ibv_pd      *pd;
        uint64_t            guid;
        int                 port_cnt;
+       int                 refcnt;
        int                 max_qpsize;
        uint8_t             max_initiator_depth;
        uint8_t             max_responder_resources;
@@ -122,6 +123,7 @@ static int cma_dev_cnt;
 static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
 static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
 int af_ib_support;
+static int verbs_refcnt;
 
 #define container_of(ptr, type, field) \
        ((type *) ((void *)ptr - offsetof(type, field)))
@@ -129,10 +131,12 @@ int af_ib_support;
 static void ucma_cleanup(void)
 {
        ucma_ib_cleanup();
+       ucma_release_verbs();
 
        if (cma_dev_cnt) {
                while (cma_dev_cnt--) {
-                       ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
+                       if (cma_dev_array[cma_dev_cnt].refcnt)
+                               ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
                        ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
                }
 
@@ -235,7 +239,7 @@ int ucma_init(void)
                goto err2;
        }
                
-       cma_dev_array = malloc(sizeof *cma_dev * dev_cnt);
+       cma_dev_array = calloc(dev_cnt, sizeof *cma_dev);
        if (!cma_dev_array) {
                ret = ERR(ENOMEM);
                goto err2;
@@ -252,13 +256,6 @@ int ucma_init(void)
                        goto err3;
                }
 
-               cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
-               if (!cma_dev->pd) {
-                       ibv_close_device(cma_dev->verbs);
-                       ret = ERR(ENOMEM);
-                       goto err3;
-               }
-
                i++;
                ret = ibv_query_device(cma_dev->verbs, &attr);
                if (ret) {
@@ -280,10 +277,8 @@ int ucma_init(void)
        return 0;
 
 err3:
-       while (i--) {
-               ibv_dealloc_pd(cma_dev_array[i].pd);
+       while (i--)
                ibv_close_device(cma_dev_array[i].verbs);
-       }
        free(cma_dev_array);
 err2:
        ibv_free_device_list(dev_list);
@@ -358,15 +353,34 @@ static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
 
        for (i = 0; i < cma_dev_cnt; i++) {
                cma_dev = &cma_dev_array[i];
-               if (cma_dev->guid == guid) {
-                       id_priv->cma_dev = cma_dev;
-                       id_priv->id.verbs = cma_dev->verbs;
-                       id_priv->id.pd = cma_dev->pd;
-                       return 0;
-               }
+               if (cma_dev->guid == guid)
+                       goto match;
        }
 
        return ERR(ENODEV);
+match:
+       pthread_mutex_lock(&mut);
+       if (!cma_dev->refcnt++) {
+               cma_dev->pd = ibv_alloc_pd(cma_dev_array[i].verbs);
+               if (!cma_dev->pd) {
+                       cma_dev->refcnt--;
+                       pthread_mutex_unlock(&mut);
+                       return ERR(ENOMEM);
+               }
+       }
+       pthread_mutex_unlock(&mut);
+       id_priv->cma_dev = cma_dev;
+       id_priv->id.verbs = cma_dev->verbs;
+       id_priv->id.pd = cma_dev->pd;
+       return 0;
+}
+
+static void ucma_put_device(struct cma_device *cma_dev)
+{
+       pthread_mutex_lock(&mut);
+       if (!--cma_dev->verbs_refcnt)
+               ibv_dealloc_pd(cma_dev_array[i].pd)
+       pthread_mutex_unlock(&mut);
 }
 
 static void ucma_free_id(struct cma_id_private *id_priv)