From: Sean Hefty Date: Wed, 12 Sep 2012 18:32:27 +0000 (-0700) Subject: libibverbs: Support older providers that do not support extensions X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=d4b9589aed445542a5386c38c66e57946adb16dc;p=~shefty%2Flibibverbs.git libibverbs: Support older providers that do not support extensions Provide a compatibility layer for providers that do not support extended structures. Since the providers may have been built against an older verbs header, we need to map between the larger structured exported by verbs and the smaller ones allocated by the provider. Signed-off-by: Sean Hefty --- diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h index a2577d8..40bb4e6 100644 --- a/include/infiniband/verbs.h +++ b/include/infiniband/verbs.h @@ -307,9 +307,14 @@ enum ibv_access_flags { IBV_ACCESS_MW_BIND = (1<<4) }; +enum ibv_pd_mask { + IBV_PD_RESERVED = 1 << 0 +}; + struct ibv_pd { struct ibv_context *context; uint32_t handle; + uint32_t comp_mask; }; enum ibv_rereg_mr_flags { @@ -319,6 +324,10 @@ enum ibv_rereg_mr_flags { IBV_REREG_MR_KEEP_VALID = (1 << 3) }; +enum ibv_mr_mask { + IBV_MR_RESERVED = 1 << 0 +}; + struct ibv_mr { struct ibv_context *context; struct ibv_pd *pd; @@ -327,6 +336,7 @@ struct ibv_mr { uint32_t handle; uint32_t lkey; uint32_t rkey; + uint32_t comp_mask; }; enum ibv_mw_type { @@ -334,10 +344,15 @@ enum ibv_mw_type { IBV_MW_TYPE_2 = 2 }; +enum ibv_mw_mask { + IBV_MW_RESERVED = 1 << 0 +}; + struct ibv_mw { struct ibv_context *context; struct ibv_pd *pd; uint32_t rkey; + uint32_t comp_mask; }; struct ibv_global_route { @@ -568,6 +583,10 @@ struct ibv_mw_bind { int mw_access_flags; }; +enum ibv_srq_mask { + IBV_SRQ_RESERVED = 1 << 0 +}; + struct ibv_srq { struct ibv_context *context; void *srq_context; @@ -577,6 +596,12 @@ struct ibv_srq { pthread_mutex_t mutex; pthread_cond_t cond; uint32_t events_completed; + + uint32_t comp_mask; +}; + +enum ibv_qp_mask { + IBV_QP_RESERVED = 1 << 0 }; struct ibv_qp { @@ -594,12 +619,24 @@ struct ibv_qp { pthread_mutex_t mutex; pthread_cond_t cond; uint32_t events_completed; + + uint32_t comp_mask; +}; + +enum ibv_comp_channel_mask { + IBV_COMP_CHANNEL_RESERVED = 1 << 0 }; struct ibv_comp_channel { struct ibv_context *context; int fd; int refcnt; + + uint32_t comp_mask; +}; + +enum ibv_cq_mask { + IBV_CQ_RESERVED = 1 << 0 }; struct ibv_cq { @@ -613,12 +650,20 @@ struct ibv_cq { pthread_cond_t cond; uint32_t comp_events_completed; uint32_t async_events_completed; + + uint32_t comp_mask; +}; + +enum ibv_ah_mask { + IBV_AH_RESERVED = 1 << 0 }; struct ibv_ah { struct ibv_context *context; struct ibv_pd *pd; uint32_t handle; + + uint32_t comp_mask; }; struct ibv_device; diff --git a/src/compat-ex.c b/src/compat-ex.c new file mode 100644 index 0000000..d822f73 --- /dev/null +++ b/src/compat-ex.c @@ -0,0 +1,398 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012 Intel Corp., Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include +#endif /* HAVE_CONFIG_H */ + +#include +#include +#include +#include +#include + +#include "ibverbs.h" + +struct _ibv_pd_ex { + struct ibv_pd pd; + struct ibv_pd *real_pd; +}; + +struct _ibv_mr_ex { + struct ibv_mr mr; + struct ibv_mr *real_mr; +}; + +struct _ibv_srq_ex { + struct ibv_srq srq; + struct ibv_srq *real_srq; +}; + +struct _ibv_qp_ex { + struct ibv_qp qp; + struct ibv_qp *real_qp; +}; + +struct _ibv_cq_ex { + struct ibv_cq cq; + struct ibv_cq *real_cq; +}; + +struct _ibv_ah_ex { + struct ibv_ah ah; + struct ibv_ah *real_ah; +}; + +struct _ibv_context_ex { + struct ibv_context context; + struct ibv_context *real_context; +}; + +static int _ibv_query_device_ex(struct ibv_context *context, + struct ibv_device_attr *device_attr) +{ + struct _ibv_context_ex *context_ex; + context_ex = container_of(context, struct _ibv_context_ex, context); + return ibv_query_device(context_ex->real_context, device_attr); +} + +static int _ibv_query_port_ex(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr) +{ + struct _ibv_context_ex *context_ex; + context_ex = container_of(context, struct _ibv_context_ex, context); + return ibv_query_port(context_ex->real_context, port_num, port_attr); +} + +static struct ibv_pd *_ibv_alloc_pd_ex(struct ibv_context *context) +{ + struct _ibv_context_ex *context_ex; + struct _ibv_pd_ex *pd_ex; + + context_ex = container_of(context, struct _ibv_context_ex, context); + pd_ex = calloc(1, sizeof *pd_ex); + if (!pd_ex) + return NULL; + + pd_ex->real_pd = ibv_alloc_pd(context_ex->real_context); + if (!pd_ex->real_pd) { + free(pd_ex); + return NULL; + } + + return &pd_ex->pd; +} + +static int _ibv_dealloc_pd_ex(struct ibv_pd *pd) +{ + struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + int ret; + + ret = ibv_dealloc_pd(pd_ex->real_pd); + if (ret) + return ret; + + free(pd_ex); + return 0; +} + +static struct ibv_mr *_ibv_reg_mr_ex(struct ibv_pd *pd, void *addr, size_t length, + int access) +{ + struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + struct _ibv_mr_ex *mr_ex; + + mr_ex = calloc(1, sizeof *mr_ex); + if (!mr_ex) + return NULL; + + mr_ex->real_mr = ibv_reg_mr(pd_ex->real_pd, addr, length, access); + if (!mr_ex->real_mr) { + free(mr_ex); + return NULL; + } + + mr_ex->mr.lkey = mr_ex->real_mr->lkey; + mr_ex->mr.rkey = mr_ex->real_mr->rkey; + return &mr_ex->mr; +} + +static int _ibv_dereg_mr_ex(struct ibv_mr *mr) +{ + struct _ibv_mr_ex *mr_ex = container_of(mr, struct _ibv_mr_ex, mr); + int ret; + + ret = ibv_dereg_mr(mr_ex->real_mr); + if (ret) + return ret; + + free(mr_ex); + return 0; +} + +/* + * ibv_create_cq holds the channel mutex while calling the provider. We + * cannot call ibv_create_cq recursively, so we need to call the provider + * directly and duplicate initializing the ibv_cq. + */ +struct ibv_cq *_ibv_create_cq_ex(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector) +{ + struct _ibv_context_ex *context_ex; + struct _ibv_cq_ex *cq_ex; + + context_ex = container_of(context, struct _ibv_context_ex, context); + cq_ex = calloc(1, sizeof *cq_ex); + if (!cq_ex) + return NULL; + + cq_ex->real_cq = context_ex->real_context->ops.create_cq(context_ex->real_context, + cqe, channel, comp_vector); + if (!cq_ex->real_cq) { + free(cq_ex); + return NULL; + } + + cq_ex->real_cq->context = context_ex->real_context; + cq_ex->real_cq->channel = channel; + if (channel) + ++channel->refcnt; + cq_ex->real_cq->cq_context = cq_ex; + cq_ex->real_cq->comp_events_completed = 0; + cq_ex->real_cq->async_events_completed = 0; + pthread_mutex_init(&cq_ex->real_cq->mutex, NULL); + pthread_cond_init(&cq_ex->real_cq->cond, NULL); + + cq_ex->cq.cqe = cq_ex->real_cq->cqe; + return &cq_ex->cq; +} + +static int _ibv_poll_cq_ex(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) +{ + struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); + return ibv_poll_cq(cq_ex->real_cq, num_entries, wc); +} + +static int _ibv_req_notify_cq_ex(struct ibv_cq *cq, int solicited_only) +{ + struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); + return ibv_req_notify_cq(cq_ex->real_cq, solicited_only); +} + +static void _ibv_cq_event_ex(struct ibv_cq *cq) +{ + struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); + struct _ibv_context_ex *context_ex; + + context_ex = container_of(cq->context, struct _ibv_context_ex, context); + return context_ex->real_context->ops.cq_event(cq_ex->real_cq); +} + +static int _ibv_resize_cq_ex(struct ibv_cq *cq, int cqe) +{ + struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); + int ret; + + ret = ibv_resize_cq(cq_ex->real_cq, cqe); + if (ret) + return ret; + + cq_ex->cq.cqe = cq_ex->real_cq->cqe; + return 0; +} + +/* + * ibv_destroy_cq holds the mutex to any corresponding channel. Call the + * provider directly. + */ +static int _ibv_destroy_cq_ex(struct ibv_cq *cq) +{ + struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); + int ret; + + ret = cq_ex->real_cq->context->ops.destroy_cq(cq_ex->real_cq); + if (ret) + return ret; + + if (cq_ex->cq.channel) + --cq_ex->cq.channel->refcnt; + + free(cq_ex); + return 0; +} + +static struct ibv_srq *_ibv_create_srq_ex(struct ibv_pd *pd, + struct ibv_srq_init_attr *srq_init_attr) +{ + struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + struct _ibv_srq_ex *srq_ex; + + srq_ex = calloc(1, sizeof *srq_ex); + if (!srq_ex) + return NULL; + + srq_ex->real_srq = ibv_create_srq(pd_ex->real_pd, srq_init_attr); + if (!srq_ex->real_srq) { + free(srq_ex); + return NULL; + } + + return &srq_ex->srq; +} + +static int _ibv_modify_srq_ex(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr, + int srq_attr_mask) +{ + struct _ibv_srq_ex *srq_ex; + srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + +} + +static int _ibv_query_srq_ex(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr) +{ + struct _ibv_srq_ex *srq_ex; + srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + +} + +static int _ibv_destroy_srq_ex(struct ibv_srq *srq) +{ + struct _ibv_srq_ex *srq_ex; + srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + +} + +static int _ibv_post_srq_recv_ex(struct ibv_srq *srq, + struct ibv_recv_wr *recv_wr, + struct ibv_recv_wr **bad_recv_wr) +{ + struct _ibv_srq_ex *srq_ex; + srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + +} + +static struct ibv_qp *_ibv_create_qp_ex(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) +{ + struct _ibv_pd_ex *pd_ex; + pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + +} + +static int _ibv_query_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask, + struct ibv_qp_init_attr *init_attr) +{ + struct _ibv_qp_ex *qp_ex; + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + +} + +static int _ibv_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask) +{ + struct _ibv_qp_ex *qp_ex; + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + +} + +static int _ibv_destroy_qp_ex(struct ibv_qp *qp) +{ + struct _ibv_qp_ex *qp_ex; + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + +} + +static int _ibv_post_send_ex(struct ibv_qp *qp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr) +{ + struct _ibv_qp_ex *qp_ex; + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + +} + +static int _ibv_post_recv_ex(struct ibv_qp *qp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr) +{ + struct _ibv_qp_ex *qp_ex; + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + +} + +static struct ibv_ah *_ibv_create_ah_ex(struct ibv_pd *pd, struct ibv_ah_attr *attr) +{ + struct _ibv_pd_ex *pd_ex; + struct _ibv_ah_ex *ah_ex; + pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + +} + +static int _ibv_destroy_ah_ex(struct ibv_ah *ah) +{ + struct _ibv_ah_ex *ah_ex; + ah_ex = container_of(ah, struct _ibv_ah_ex, ah); + +} + +static int _ibv_attach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid, + uint16_t lid) +{ + struct _ibv_qp_ex *qp_ex; + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + +} + +static int _ibv_detach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid, + uint16_t lid) +{ + struct _ibv_qp_ex *qp_ex; + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + +} + +static void _ibv_async_event_ex(struct ibv_async_event *event) +{ + +} + +struct ibv_context *_ibv_open_device_ex(struct ibv_device *device) +{ + +} + +int _ibv_close_device_ex(struct ibv_context *context) +{ + +}