From 2ffda7f2991395570b9e776ff5ae256ca9684771 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Thu, 19 Jul 2012 10:09:48 -0700 Subject: [PATCH] librdmacm: Only allocate verbs resources when needed The librdmacm allocates a PD per device on initialization. Although we need to maintain the device list while the library is loaded (see rdma_get_devices), we can reduce the overhead by only allocating verbs resources when they are needed. This allows the rsocket preload library to support fork for applications that spawn connections off to child processes. Signed-off-by: Sean Hefty --- src/cma.c | 53 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/cma.c b/src/cma.c index bad8bce3..5280620e 100755 --- a/src/cma.c +++ b/src/cma.c @@ -76,6 +76,7 @@ struct cma_device { struct ibv_pd *pd; uint64_t guid; int port_cnt; + int refcnt; int max_qpsize; uint8_t max_initiator_depth; uint8_t max_responder_resources; @@ -132,7 +133,8 @@ static void ucma_cleanup(void) if (cma_dev_cnt) { while (cma_dev_cnt--) { - ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd); + if (cma_dev_array[cma_dev_cnt].refcnt) + ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd); ibv_close_device(cma_dev_array[cma_dev_cnt].verbs); } @@ -235,7 +237,7 @@ int ucma_init(void) goto err2; } - cma_dev_array = malloc(sizeof *cma_dev * dev_cnt); + cma_dev_array = calloc(dev_cnt, sizeof *cma_dev); if (!cma_dev_array) { ret = ERR(ENOMEM); goto err2; @@ -252,13 +254,6 @@ int ucma_init(void) goto err3; } - cma_dev->pd = ibv_alloc_pd(cma_dev->verbs); - if (!cma_dev->pd) { - ibv_close_device(cma_dev->verbs); - ret = ERR(ENOMEM); - goto err3; - } - i++; ret = ibv_query_device(cma_dev->verbs, &attr); if (ret) { @@ -280,10 +275,8 @@ int ucma_init(void) return 0; err3: - while (i--) { - ibv_dealloc_pd(cma_dev_array[i].pd); + while (i--) ibv_close_device(cma_dev_array[i].verbs); - } free(cma_dev_array); err2: ibv_free_device_list(dev_list); @@ -354,23 +347,45 @@ void rdma_destroy_event_channel(struct rdma_event_channel *channel) static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid) { struct cma_device *cma_dev; - int i; + int i, ret = 0; for (i = 0; i < cma_dev_cnt; i++) { cma_dev = &cma_dev_array[i]; - if (cma_dev->guid == guid) { - id_priv->cma_dev = cma_dev; - id_priv->id.verbs = cma_dev->verbs; - id_priv->id.pd = cma_dev->pd; - return 0; - } + if (cma_dev->guid == guid) + goto match; } return ERR(ENODEV); +match: + pthread_mutex_lock(&mut); + if (!cma_dev->refcnt++) { + cma_dev->pd = ibv_alloc_pd(cma_dev_array[i].verbs); + if (!cma_dev->pd) { + cma_dev->refcnt--; + ret = ERR(ENOMEM); + goto out; + } + } + id_priv->cma_dev = cma_dev; + id_priv->id.verbs = cma_dev->verbs; + id_priv->id.pd = cma_dev->pd; +out: + pthread_mutex_unlock(&mut); + return ret; +} + +static void ucma_put_device(struct cma_device *cma_dev) +{ + pthread_mutex_lock(&mut); + if (!--cma_dev->refcnt) + ibv_dealloc_pd(cma_dev->pd); + pthread_mutex_unlock(&mut); } static void ucma_free_id(struct cma_id_private *id_priv) { + if (id_priv->cma_dev) + ucma_put_device(id_priv->cma_dev); pthread_cond_destroy(&id_priv->cond); pthread_mutex_destroy(&id_priv->mut); if (id_priv->id.route.path_rec) -- 2.46.0