librdmacm: Only allocate verbs resources when needed

The librdmacm allocates a PD per device on initialization.  Although
we need to maintain the device list while the library is loaded
(see rdma_get_devices), we can reduce the overhead by only allocating
verbs resources when they are needed.

This allows the rsocket preload library to support fork for
applications that spawn connections off to child processes.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/src/cma.c b/src/cma.c
index bad8bce32e0c91590a0c1b921bc199c4eb81c97b..65b52bf7da5c4ca479611b7ac25f996370096425 100755 (executable)
--- a/src/cma.c
+++ b/src/cma.c
@@ -76,6 +76,7 @@ struct cma_device {
        struct ibv_pd      *pd;
        uint64_t            guid;
        int                 port_cnt;
+       int                 refcnt;
        int                 max_qpsize;
        uint8_t             max_initiator_depth;
        uint8_t             max_responder_resources;
@@ -122,6 +123,7 @@ static int cma_dev_cnt;
 static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
 static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
 int af_ib_support;
+static int verbs_refcnt;
 
 #define container_of(ptr, type, field) \
        ((type *) ((void *)ptr - offsetof(type, field)))
@@ -132,7 +134,8 @@ static void ucma_cleanup(void)
 
        if (cma_dev_cnt) {
                while (cma_dev_cnt--) {
-                       ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
+                       if (cma_dev_array[cma_dev_cnt].refcnt)
+                               ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
                        ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
                }
 
@@ -235,7 +238,7 @@ int ucma_init(void)
                goto err2;
        }
                
-       cma_dev_array = malloc(sizeof *cma_dev * dev_cnt);
+       cma_dev_array = calloc(dev_cnt, sizeof *cma_dev);
        if (!cma_dev_array) {
                ret = ERR(ENOMEM);
                goto err2;
@@ -252,13 +255,6 @@ int ucma_init(void)
                        goto err3;
                }
 
-               cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
-               if (!cma_dev->pd) {
-                       ibv_close_device(cma_dev->verbs);
-                       ret = ERR(ENOMEM);
-                       goto err3;
-               }
-
                i++;
                ret = ibv_query_device(cma_dev->verbs, &attr);
                if (ret) {
@@ -280,10 +276,8 @@ int ucma_init(void)
        return 0;
 
 err3:
-       while (i--) {
-               ibv_dealloc_pd(cma_dev_array[i].pd);
+       while (i--)
                ibv_close_device(cma_dev_array[i].verbs);
-       }
        free(cma_dev_array);
 err2:
        ibv_free_device_list(dev_list);
@@ -354,23 +348,45 @@ void rdma_destroy_event_channel(struct rdma_event_channel *channel)
 static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
 {
        struct cma_device *cma_dev;
-       int i;
+       int i, ret = 0;
 
        for (i = 0; i < cma_dev_cnt; i++) {
                cma_dev = &cma_dev_array[i];
-               if (cma_dev->guid == guid) {
-                       id_priv->cma_dev = cma_dev;
-                       id_priv->id.verbs = cma_dev->verbs;
-                       id_priv->id.pd = cma_dev->pd;
-                       return 0;
-               }
+               if (cma_dev->guid == guid)
+                       goto match;
        }
 
        return ERR(ENODEV);
+match:
+       pthread_mutex_lock(&mut);
+       if (!cma_dev->refcnt++) {
+               cma_dev->pd = ibv_alloc_pd(cma_dev_array[i].verbs);
+               if (!cma_dev->pd) {
+                       cma_dev->refcnt--;
+                       ret = ERR(ENOMEM);
+                       goto out;
+               }
+       }
+       id_priv->cma_dev = cma_dev;
+       id_priv->id.verbs = cma_dev->verbs;
+       id_priv->id.pd = cma_dev->pd;
+out:
+       pthread_mutex_unlock(&mut);
+       return ret;
+}
+
+static void ucma_put_device(struct cma_device *cma_dev)
+{
+       pthread_mutex_lock(&mut);
+       if (!--cma_dev->refcnt)
+               ibv_dealloc_pd(cma_dev->pd);
+       pthread_mutex_unlock(&mut);
 }
 
 static void ucma_free_id(struct cma_id_private *id_priv)
 {
+       if (id_priv->cma_dev)
+               ucma_put_device(id_priv->cma_dev);
        pthread_cond_destroy(&id_priv->cond);
        pthread_mutex_destroy(&id_priv->mut);
        if (id_priv->id.route.path_rec)