libibverbs: Support older providers that do not support extensions

Provide a compatibility layer for providers that do not support
extended structures.  Since the providers may have been built
against an older verbs header, we need to map between the larger
structured exported by verbs and the smaller ones allocated by
the provider.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index a2577d87ddc9a86973367675f546ffc8eb6e836e..40bb4e66734c3a55e43a846ed69a59ddfc368d0d 100644 (file)
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -307,9 +307,14 @@ enum ibv_access_flags {
        IBV_ACCESS_MW_BIND              = (1<<4)
 };
 
+enum ibv_pd_mask {
+       IBV_PD_RESERVED                 = 1 << 0
+};
+
 struct ibv_pd {
        struct ibv_context     *context;
        uint32_t                handle;
+       uint32_t                comp_mask;
 };
 
 enum ibv_rereg_mr_flags {
@@ -319,6 +324,10 @@ enum ibv_rereg_mr_flags {
        IBV_REREG_MR_KEEP_VALID         = (1 << 3)
 };
 
+enum ibv_mr_mask {
+       IBV_MR_RESERVED                 = 1 << 0
+};
+
 struct ibv_mr {
        struct ibv_context     *context;
        struct ibv_pd          *pd;
@@ -327,6 +336,7 @@ struct ibv_mr {
        uint32_t                handle;
        uint32_t                lkey;
        uint32_t                rkey;
+       uint32_t                comp_mask;
 };
 
 enum ibv_mw_type {
@@ -334,10 +344,15 @@ enum ibv_mw_type {
        IBV_MW_TYPE_2                   = 2
 };
 
+enum ibv_mw_mask {
+       IBV_MW_RESERVED                 = 1 << 0
+};
+
 struct ibv_mw {
        struct ibv_context     *context;
        struct ibv_pd          *pd;
        uint32_t                rkey;
+       uint32_t                comp_mask;
 };
 
 struct ibv_global_route {
@@ -568,6 +583,10 @@ struct ibv_mw_bind {
        int                     mw_access_flags;
 };
 
+enum ibv_srq_mask {
+       IBV_SRQ_RESERVED                = 1 << 0
+};
+
 struct ibv_srq {
        struct ibv_context     *context;
        void                   *srq_context;
@@ -577,6 +596,12 @@ struct ibv_srq {
        pthread_mutex_t         mutex;
        pthread_cond_t          cond;
        uint32_t                events_completed;
+
+       uint32_t                comp_mask;
+};
+
+enum ibv_qp_mask {
+       IBV_QP_RESERVED                 = 1 << 0
 };
 
 struct ibv_qp {
@@ -594,12 +619,24 @@ struct ibv_qp {
        pthread_mutex_t         mutex;
        pthread_cond_t          cond;
        uint32_t                events_completed;
+
+       uint32_t                comp_mask;
+};
+
+enum ibv_comp_channel_mask {
+       IBV_COMP_CHANNEL_RESERVED       = 1 << 0
 };
 
 struct ibv_comp_channel {
        struct ibv_context     *context;
        int                     fd;
        int                     refcnt;
+
+       uint32_t                comp_mask;
+};
+
+enum ibv_cq_mask {
+       IBV_CQ_RESERVED                 = 1 << 0
 };
 
 struct ibv_cq {
@@ -613,12 +650,20 @@ struct ibv_cq {
        pthread_cond_t          cond;
        uint32_t                comp_events_completed;
        uint32_t                async_events_completed;
+
+       uint32_t                comp_mask;
+};
+
+enum ibv_ah_mask {
+       IBV_AH_RESERVED                 = 1 << 0
 };
 
 struct ibv_ah {
        struct ibv_context     *context;
        struct ibv_pd          *pd;
        uint32_t                handle;
+
+       uint32_t                comp_mask;
 };
 
 struct ibv_device;

diff --git a/src/compat-ex.c b/src/compat-ex.c
new file mode 100644 (file)
index 0000000..d822f73
--- /dev/null
+++ b/src/compat-ex.c
@@ -0,0 +1,398 @@
+/*
+ * Copyright (c) 2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012 Intel Corp., Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <string.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <alloca.h>
+
+#include "ibverbs.h"
+
+struct _ibv_pd_ex {
+       struct ibv_pd  pd;
+       struct ibv_pd *real_pd;
+};
+
+struct _ibv_mr_ex {
+       struct ibv_mr  mr;
+       struct ibv_mr *real_mr;
+};
+
+struct _ibv_srq_ex {
+       struct ibv_srq  srq;
+       struct ibv_srq *real_srq;
+};
+
+struct _ibv_qp_ex {
+       struct ibv_qp  qp;
+       struct ibv_qp *real_qp;
+};
+
+struct _ibv_cq_ex {
+       struct ibv_cq  cq;
+       struct ibv_cq *real_cq;
+};
+
+struct _ibv_ah_ex {
+       struct ibv_ah  ah;
+       struct ibv_ah *real_ah;
+};
+
+struct _ibv_context_ex {
+       struct ibv_context  context;
+       struct ibv_context *real_context;
+};
+
+static int _ibv_query_device_ex(struct ibv_context *context,
+                               struct ibv_device_attr *device_attr)
+{
+       struct _ibv_context_ex *context_ex;
+       context_ex = container_of(context, struct _ibv_context_ex, context);
+       return ibv_query_device(context_ex->real_context, device_attr);
+}
+
+static int _ibv_query_port_ex(struct ibv_context *context, uint8_t port_num,
+                             struct ibv_port_attr *port_attr)
+{
+       struct _ibv_context_ex *context_ex;
+       context_ex = container_of(context, struct _ibv_context_ex, context);
+       return ibv_query_port(context_ex->real_context, port_num, port_attr);
+}
+
+static struct ibv_pd *_ibv_alloc_pd_ex(struct ibv_context *context)
+{
+       struct _ibv_context_ex *context_ex;
+       struct _ibv_pd_ex *pd_ex;
+
+       context_ex = container_of(context, struct _ibv_context_ex, context);
+       pd_ex = calloc(1, sizeof *pd_ex);
+       if (!pd_ex)
+               return NULL;
+
+       pd_ex->real_pd = ibv_alloc_pd(context_ex->real_context);
+       if (!pd_ex->real_pd) {
+               free(pd_ex);
+               return NULL;
+       }
+
+       return &pd_ex->pd;
+}
+
+static int _ibv_dealloc_pd_ex(struct ibv_pd *pd)
+{
+       struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd);
+       int ret;
+
+       ret = ibv_dealloc_pd(pd_ex->real_pd);
+       if (ret)
+               return ret;
+
+       free(pd_ex);
+       return 0;
+}
+
+static struct ibv_mr *_ibv_reg_mr_ex(struct ibv_pd *pd, void *addr, size_t length,
+                                    int access)
+{
+       struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd);
+       struct _ibv_mr_ex *mr_ex;
+
+       mr_ex = calloc(1, sizeof *mr_ex);
+       if (!mr_ex)
+               return NULL;
+
+       mr_ex->real_mr = ibv_reg_mr(pd_ex->real_pd, addr, length, access);
+       if (!mr_ex->real_mr) {
+               free(mr_ex);
+               return NULL;
+       }
+
+       mr_ex->mr.lkey = mr_ex->real_mr->lkey;
+       mr_ex->mr.rkey = mr_ex->real_mr->rkey;
+       return &mr_ex->mr;
+}
+
+static int _ibv_dereg_mr_ex(struct ibv_mr *mr)
+{
+       struct _ibv_mr_ex *mr_ex = container_of(mr, struct _ibv_mr_ex, mr);
+       int ret;
+
+       ret = ibv_dereg_mr(mr_ex->real_mr);
+       if (ret)
+               return ret;
+
+       free(mr_ex);
+       return 0;
+}
+
+/*
+ * ibv_create_cq holds the channel mutex while calling the provider.  We
+ * cannot call ibv_create_cq recursively, so we need to call the provider
+ * directly and duplicate initializing the ibv_cq.
+ */
+struct ibv_cq *_ibv_create_cq_ex(struct ibv_context *context, int cqe,
+                                struct ibv_comp_channel *channel,
+                                int comp_vector)
+{
+       struct _ibv_context_ex *context_ex;
+       struct _ibv_cq_ex *cq_ex;
+
+       context_ex = container_of(context, struct _ibv_context_ex, context);
+       cq_ex = calloc(1, sizeof *cq_ex);
+       if (!cq_ex)
+               return NULL;
+
+       cq_ex->real_cq = context_ex->real_context->ops.create_cq(context_ex->real_context,
+                                                                cqe, channel, comp_vector);
+       if (!cq_ex->real_cq) {
+               free(cq_ex);
+               return NULL;
+       }
+
+       cq_ex->real_cq->context = context_ex->real_context;
+       cq_ex->real_cq->channel = channel;
+       if (channel)
+               ++channel->refcnt;
+       cq_ex->real_cq->cq_context = cq_ex;
+       cq_ex->real_cq->comp_events_completed  = 0;
+       cq_ex->real_cq->async_events_completed = 0;
+       pthread_mutex_init(&cq_ex->real_cq->mutex, NULL);
+       pthread_cond_init(&cq_ex->real_cq->cond, NULL);
+
+       cq_ex->cq.cqe = cq_ex->real_cq->cqe;
+       return &cq_ex->cq;
+}
+
+static int _ibv_poll_cq_ex(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc)
+{
+       struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq);
+       return ibv_poll_cq(cq_ex->real_cq, num_entries, wc);
+}
+
+static int _ibv_req_notify_cq_ex(struct ibv_cq *cq, int solicited_only)
+{
+       struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq);
+       return ibv_req_notify_cq(cq_ex->real_cq, solicited_only);
+}
+
+static void _ibv_cq_event_ex(struct ibv_cq *cq)
+{
+       struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq);
+       struct _ibv_context_ex *context_ex;
+
+       context_ex = container_of(cq->context, struct _ibv_context_ex, context);
+       return context_ex->real_context->ops.cq_event(cq_ex->real_cq);
+}
+
+static int _ibv_resize_cq_ex(struct ibv_cq *cq, int cqe)
+{
+       struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq);
+       int ret;
+
+       ret = ibv_resize_cq(cq_ex->real_cq, cqe);
+       if (ret)
+               return ret;
+
+       cq_ex->cq.cqe = cq_ex->real_cq->cqe;
+       return 0;
+}
+
+/*
+ * ibv_destroy_cq holds the mutex to any corresponding channel.  Call the
+ * provider directly.
+ */
+static int _ibv_destroy_cq_ex(struct ibv_cq *cq)
+{
+       struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq);
+       int ret;
+
+       ret = cq_ex->real_cq->context->ops.destroy_cq(cq_ex->real_cq);
+       if (ret)
+               return ret;
+
+       if (cq_ex->cq.channel)
+               --cq_ex->cq.channel->refcnt;
+
+       free(cq_ex);
+       return 0;
+}
+
+static struct ibv_srq *_ibv_create_srq_ex(struct ibv_pd *pd,
+                                         struct ibv_srq_init_attr *srq_init_attr)
+{
+       struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd);
+       struct _ibv_srq_ex *srq_ex;
+
+       srq_ex = calloc(1, sizeof *srq_ex);
+       if (!srq_ex)
+               return NULL;
+
+       srq_ex->real_srq = ibv_create_srq(pd_ex->real_pd, srq_init_attr);
+       if (!srq_ex->real_srq) {
+               free(srq_ex);
+               return NULL;
+       }
+
+       return &srq_ex->srq;
+}
+
+static int _ibv_modify_srq_ex(struct ibv_srq *srq,
+                             struct ibv_srq_attr *srq_attr,
+                             int srq_attr_mask)
+{
+       struct _ibv_srq_ex *srq_ex;
+       srq_ex = container_of(srq, struct _ibv_srq_ex, srq);
+
+}
+
+static int _ibv_query_srq_ex(struct ibv_srq *srq,
+                            struct ibv_srq_attr *srq_attr)
+{
+       struct _ibv_srq_ex *srq_ex;
+       srq_ex = container_of(srq, struct _ibv_srq_ex, srq);
+
+}
+
+static int _ibv_destroy_srq_ex(struct ibv_srq *srq)
+{
+       struct _ibv_srq_ex *srq_ex;
+       srq_ex = container_of(srq, struct _ibv_srq_ex, srq);
+
+}
+
+static int  _ibv_post_srq_recv_ex(struct ibv_srq *srq,
+                                 struct ibv_recv_wr *recv_wr,
+                                 struct ibv_recv_wr **bad_recv_wr)
+{
+       struct _ibv_srq_ex *srq_ex;
+       srq_ex = container_of(srq, struct _ibv_srq_ex, srq);
+
+}
+
+static struct ibv_qp *_ibv_create_qp_ex(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
+{
+       struct _ibv_pd_ex *pd_ex;
+       pd_ex = container_of(pd, struct _ibv_pd_ex, pd);
+
+}
+
+static int _ibv_query_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+                           int attr_mask,
+                            struct ibv_qp_init_attr *init_attr)
+{
+       struct _ibv_qp_ex *qp_ex;
+       qp_ex = container_of(qp, struct _ibv_qp_ex, qp);
+
+}
+
+static int _ibv_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+                            int attr_mask)
+{
+       struct _ibv_qp_ex *qp_ex;
+       qp_ex = container_of(qp, struct _ibv_qp_ex, qp);
+
+}
+
+static int _ibv_destroy_qp_ex(struct ibv_qp *qp)
+{
+       struct _ibv_qp_ex *qp_ex;
+       qp_ex = container_of(qp, struct _ibv_qp_ex, qp);
+
+}
+
+static int _ibv_post_send_ex(struct ibv_qp *qp, struct ibv_send_wr *wr,
+                            struct ibv_send_wr **bad_wr)
+{
+       struct _ibv_qp_ex *qp_ex;
+       qp_ex = container_of(qp, struct _ibv_qp_ex, qp);
+
+}
+
+static int _ibv_post_recv_ex(struct ibv_qp *qp, struct ibv_recv_wr *wr,
+                            struct ibv_recv_wr **bad_wr)
+{
+       struct _ibv_qp_ex *qp_ex;
+       qp_ex = container_of(qp, struct _ibv_qp_ex, qp);
+
+}
+
+static struct ibv_ah *_ibv_create_ah_ex(struct ibv_pd *pd, struct ibv_ah_attr *attr)
+{
+       struct _ibv_pd_ex *pd_ex;
+       struct _ibv_ah_ex *ah_ex;
+       pd_ex = container_of(pd, struct _ibv_pd_ex, pd);
+
+}
+
+static int _ibv_destroy_ah_ex(struct ibv_ah *ah)
+{
+       struct _ibv_ah_ex *ah_ex;
+       ah_ex = container_of(ah, struct _ibv_ah_ex, ah);
+
+}
+
+static int _ibv_attach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid,
+                               uint16_t lid)
+{
+       struct _ibv_qp_ex *qp_ex;
+       qp_ex = container_of(qp, struct _ibv_qp_ex, qp);
+
+}
+
+static int _ibv_detach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid,
+                               uint16_t lid)
+{
+       struct _ibv_qp_ex *qp_ex;
+       qp_ex = container_of(qp, struct _ibv_qp_ex, qp);
+
+}
+
+static void _ibv_async_event_ex(struct ibv_async_event *event)
+{
+
+}
+
+struct ibv_context *_ibv_open_device_ex(struct ibv_device *device)
+{
+
+}
+
+int _ibv_close_device_ex(struct ibv_context *context)
+{
+
+}