Reference count completion channels

Keep a reference count in completion channel structure, so that
ibv_destroy_comp_channel() can return EBUSY if a consumer tries to
destroy a channel that still has CQs attached.

Suggested by Dotan Barak <dotanb@mellanox.co.il>.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 2ae50abd31a12ac5d7a7c8c7e24e103af1cd04dc..acc1b8246608a84722d96b7d91fedc242e0081a8 100644 (file)
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -573,11 +573,14 @@ struct ibv_qp {
 };
 
 struct ibv_comp_channel {
+       struct ibv_context     *context;
        int                     fd;
+       int                     refcnt;
 };
 
 struct ibv_cq {
        struct ibv_context     *context;
+       struct ibv_comp_channel *channel;
        void                   *cq_context;
        uint32_t                handle;
        int                     cqe;
@@ -680,12 +683,13 @@ struct ibv_context_ops {
 };
 
 struct ibv_context {
-       struct ibv_device         *device;
-       struct ibv_context_ops     ops;
-       int                        cmd_fd;
-       int                        async_fd;
-       int                        num_comp_vectors;
-       void                      *abi_compat;
+       struct ibv_device      *device;
+       struct ibv_context_ops  ops;
+       int                     cmd_fd;
+       int                     async_fd;
+       int                     num_comp_vectors;
+       pthread_mutex_t         mutex;
+       void                   *abi_compat;
 };
 
 /**

diff --git a/src/cmd.c b/src/cmd.c
index f7d3fdef08e1c0dd86d571e7af93e436060874ad..a0bfaad3de743a87f9a5d7b68d7d47ffdddf63cc 100644 (file)
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -75,7 +75,9 @@ static int ibv_cmd_get_context_v2(struct ibv_context *context,
 
        context->async_fd         = resp->async_fd;
        context->num_comp_vectors = 1;
+       t->channel.context        = context;
        t->channel.fd             = cq_fd;
+       t->channel.refcnt         = 0;
        context->abi_compat       = t;
 
        return 0;

diff --git a/src/device.c b/src/device.c
index bca1ce9a8fbdfcf91f179bf5ba4bbd59af342022..3abc1ebca7fc1f0e1714281e3ca753f9a053c06c 100644 (file)
--- a/src/device.c
+++ b/src/device.c
@@ -138,6 +138,7 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
 
        context->device = device;
        context->cmd_fd = cmd_fd;
+       pthread_mutex_init(&context->mutex, NULL);
 
        return context;
 

diff --git a/src/verbs.c b/src/verbs.c
index 56513e40d6ecde1010e1e85d32b50e0e8c92c6ef..febf32aeab50b57afdcfcea4ac883213932dfa8f 100644 (file)
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -226,7 +226,9 @@ struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context)
                return NULL;
        }
 
-       channel->fd = resp.fd;
+       channel->context = context;
+       channel->fd      = resp.fd;
+       channel->refcnt  = 0;
 
        return channel;
 }
@@ -240,23 +242,46 @@ static int ibv_destroy_comp_channel_v2(struct ibv_comp_channel *channel)
 
 int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
 {
-       if (abi_ver <= 2)
-               return ibv_destroy_comp_channel_v2(channel);
+       struct ibv_context *context;
+       int ret;
+
+       context = channel->context;
+       pthread_mutex_lock(&context->mutex);
+
+       if (channel->refcnt) {
+               ret = EBUSY;
+               goto out;
+       }
+
+       if (abi_ver <= 2) {
+               ret = ibv_destroy_comp_channel_v2(channel);
+               goto out;
+       }
 
        close(channel->fd);
        free(channel);
+       ret = 0;
 
-       return 0;
+out:
+       pthread_mutex_unlock(&context->mutex);
+
+       return ret;
 }
 
 struct ibv_cq *__ibv_create_cq(struct ibv_context *context, int cqe, void *cq_context,
                               struct ibv_comp_channel *channel, int comp_vector)
 {
-       struct ibv_cq *cq = context->ops.create_cq(context, cqe, channel,
-                                                  comp_vector);
+       struct ibv_cq *cq;
+
+       pthread_mutex_lock(&context->mutex);
+
+       cq = context->ops.create_cq(context, cqe, channel, comp_vector);
 
        if (cq) {
                cq->context                = context;
+               cq->channel                = channel;
+               if (channel)
+                       ++channel->refcnt;
                cq->cq_context             = cq_context;
                cq->comp_events_completed  = 0;
                cq->async_events_completed = 0;
@@ -264,6 +289,8 @@ struct ibv_cq *__ibv_create_cq(struct ibv_context *context, int cqe, void *cq_co
                pthread_cond_init(&cq->cond, NULL);
        }
 
+       pthread_mutex_unlock(&context->mutex);
+
        return cq;
 }
 default_symver(__ibv_create_cq, ibv_create_cq);
@@ -279,7 +306,21 @@ default_symver(__ibv_resize_cq, ibv_resize_cq);
 
 int __ibv_destroy_cq(struct ibv_cq *cq)
 {
-       return cq->context->ops.destroy_cq(cq);
+       struct ibv_comp_channel *channel = cq->channel;
+       int ret;
+
+       if (channel)
+               pthread_mutex_lock(&channel->context->mutex);
+
+       ret = cq->context->ops.destroy_cq(cq);
+
+       if (channel) {
+               if (!ret)
+                       --channel->refcnt;
+               pthread_mutex_unlock(&channel->context->mutex);
+       }
+
+       return ret;
 }
 default_symver(__ibv_destroy_cq, ibv_destroy_cq);