From dc1d237058f82ad6a7d3b279e6aea4888c976e1a Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Thu, 13 Sep 2012 16:41:00 -0700 Subject: [PATCH] Refresh of compat-ex --- src/compat-ex.c | 221 +++++++++++++++++++++++++++++++++++++----------- src/device.c | 17 +++- src/ibverbs.h | 7 ++ src/verbs.c | 9 +- 4 files changed, 199 insertions(+), 55 deletions(-) diff --git a/src/compat-ex.c b/src/compat-ex.c index d822f73..5b15be0 100644 --- a/src/compat-ex.c +++ b/src/compat-ex.c @@ -78,6 +78,54 @@ struct _ibv_context_ex { struct ibv_context *real_context; }; +struct ibv_srq *_ibv_real_srq(struct ibv_srq *srq) +{ + struct _ibv_srq_ex *srq_ex; + + if (verbs_get_ctx(srq)) + return srq; + + srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + return srq_ex->real_srq; +} + +struct ibv_qp *_ibv_real_qp(struct ibv_qp *qp) +{ + struct _ibv_qp_ex *qp_ex; + + if (verbs_get_ctx(qp)) + return qp; + + qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + return qp_ex->real_qp; +} + +struct ibv_cq *_ibv_real_cq(struct ibv_cq *cq) +{ + struct _ibv_cq_ex *cq_ex; + + if (verbs_get_ctx(cq)) + return cq; + + cq_ex = container_of(cq, struct _ibv_cq_ex, cq); + return cq_ex->real_cq; +} + +struct ibv_srq *_ibv_verbs_srq(struct ibv_srq *srq) +{ + return verbs_get_ctx(srq) ? srq : (struct ibv_srq *) srq->srq_context; +} + +struct ibv_qp *_ibv_verbs_qp(struct ibv_qp *qp) +{ + return verbs_get_ctx(qp) ? qp : (struct ibv_qp *) qp->qp_context; +} + +struct ibv_cq *_ibv_verbs_cq(struct ibv_cq *cq) +{ + return verbs_get_ctx(cq) ? cq : (struct ibv_cq *) cq->cq_context; +} + static int _ibv_query_device_ex(struct ibv_context *context, struct ibv_device_attr *device_attr) { @@ -204,19 +252,17 @@ static int _ibv_poll_cq_ex(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc return ibv_poll_cq(cq_ex->real_cq, num_entries, wc); } -static int _ibv_req_notify_cq_ex(struct ibv_cq *cq, int solicited_only) +void _ibv_cq_event_ex(struct ibv_cq **cq, void **cq_context) { - struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); - return ibv_req_notify_cq(cq_ex->real_cq, solicited_only); + struct _ibv_cq_ex *cq_ex = container_of(*cq_context, struct _ibv_cq_ex, cq); + *cq = &cq_ex->cq; + *cq_context = cq_ex->cq.cq_context; } -static void _ibv_cq_event_ex(struct ibv_cq *cq) +static int _ibv_req_notify_cq_ex(struct ibv_cq *cq, int solicited_only) { struct _ibv_cq_ex *cq_ex = container_of(cq, struct _ibv_cq_ex, cq); - struct _ibv_context_ex *context_ex; - - context_ex = container_of(cq->context, struct _ibv_context_ex, context); - return context_ex->real_context->ops.cq_event(cq_ex->real_cq); + return ibv_req_notify_cq(cq_ex->real_cq, solicited_only); } static int _ibv_resize_cq_ex(struct ibv_cq *cq, int cqe) @@ -275,124 +321,201 @@ static int _ibv_modify_srq_ex(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, int srq_attr_mask) { - struct _ibv_srq_ex *srq_ex; - srq_ex = container_of(srq, struct _ibv_srq_ex, srq); - + struct _ibv_srq_ex *srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + return ibv_modify_srq(srq_ex->real_srq, srq_attr, srq_attr_mask); } static int _ibv_query_srq_ex(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr) { - struct _ibv_srq_ex *srq_ex; - srq_ex = container_of(srq, struct _ibv_srq_ex, srq); - + struct _ibv_srq_ex *srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + return ibv_query_srq(srq_ex->real_srq, srq_attr); } static int _ibv_destroy_srq_ex(struct ibv_srq *srq) { - struct _ibv_srq_ex *srq_ex; - srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + struct _ibv_srq_ex *srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + int ret; + ret = ibv_destroy_srq(srq_ex->real_srq); + if (ret) + return ret; + + free(srq_ex); + return 0; } -static int _ibv_post_srq_recv_ex(struct ibv_srq *srq, - struct ibv_recv_wr *recv_wr, - struct ibv_recv_wr **bad_recv_wr) +static int _ibv_post_srq_recv_ex(struct ibv_srq *srq, + struct ibv_recv_wr *recv_wr, + struct ibv_recv_wr **bad_recv_wr) { - struct _ibv_srq_ex *srq_ex; - srq_ex = container_of(srq, struct _ibv_srq_ex, srq); - + struct _ibv_srq_ex *srq_ex = container_of(srq, struct _ibv_srq_ex, srq); + return ibv_post_srq_recv(srq_ex->real_srq, recv_wr, bad_recv_wr); } static struct ibv_qp *_ibv_create_qp_ex(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) { - struct _ibv_pd_ex *pd_ex; - pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + struct _ibv_qp_ex *qp_ex; + struct _ibv_cq_ex *scq_ex, *rcq_ex; + struct _ibv_srq_ex *srq_ex; + struct ibv_qp_init_attr real_attr; + + qp_ex = calloc(1, sizeof *qp_ex); + if (!qp_ex) + return NULL; + scq_ex = container_of(attr->send_cq, struct _ibv_cq_ex, cq); + rcq_ex = container_of(attr->recv_cq, struct _ibv_cq_ex, cq); + srq_ex = attr->srq ? container_of(attr->srq, struct _ibv_srq_ex, srq) : NULL; + + real_attr.qp_context = qp_ex; + real_attr.send_cq = scq_ex->real_cq; + real_attr.recv_cq = rcq_ex->real_cq; + real_attr.srq = srq_ex->real_srq; + real_attr.cap = attr->cap; + real_attr.qp_type = attr->qp_type; + real_attr.sq_sig_all = attr->sq_sig_all; + + qp_ex->real_qp = ibv_create_qp(pd_ex->real_pd, &real_attr); + if (!qp_ex->real_qp) { + free(qp_ex); + return NULL; + } + + return &qp_ex->qp; } static int _ibv_query_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_qp_init_attr *init_attr) + int attr_mask, struct ibv_qp_init_attr *init_attr) { - struct _ibv_qp_ex *qp_ex; - qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + int ret; + ret = ibv_query_qp(qp_ex->real_qp, attr, attr_mask, init_attr); + if (ret) + return ret; + + init_attr->qp_context = qp_ex->qp.qp_context; + init_attr->send_cq = qp_ex->qp.send_cq; + init_attr->recv_cq = qp_ex->qp.recv_cq; + init_attr->srq = qp_ex->qp.srq; + return 0; } static int _ibv_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) { - struct _ibv_qp_ex *qp_ex; - qp_ex = container_of(qp, struct _ibv_qp_ex, qp); - + struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + return ibv_modify_qp(qp_ex->real_qp, attr, attr_mask); } static int _ibv_destroy_qp_ex(struct ibv_qp *qp) { - struct _ibv_qp_ex *qp_ex; - qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + int ret; + ret = ibv_destroy_qp(qp_ex->real_qp); + if (ret) + return ret; + + free(qp_ex); + return 0; } static int _ibv_post_send_ex(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr) { - struct _ibv_qp_ex *qp_ex; - qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + struct _ibv_ah_ex *ah_ex; + struct ibv_send_wr real_wr, *cur_wr; + int ret; + if (qp->qp_type != IBV_QPT_UD) + return ibv_post_send(qp_ex->real_qp, wr, bad_wr); + + for (cur_wr = wr, ret = 0; cur_wr && !ret; cur_wr = cur_wr->next) { + real_wr = *cur_wr; + real_wr.next = NULL; + ah_ex = container_of(cur_wr->wr.ud.ah, struct _ibv_ah_ex, ah); + real_wr.wr.ud.ah = ah_ex->real_ah; + ret = ibv_post_send(qp_ex->real_qp, &real_wr, bad_wr); + } + *bad_wr = cur_wr; + return ret; } static int _ibv_post_recv_ex(struct ibv_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) { - struct _ibv_qp_ex *qp_ex; - qp_ex = container_of(qp, struct _ibv_qp_ex, qp); - + struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + return ibv_post_recv(qp_ex->real_qp, wr, bad_wr); } static struct ibv_ah *_ibv_create_ah_ex(struct ibv_pd *pd, struct ibv_ah_attr *attr) { - struct _ibv_pd_ex *pd_ex; + struct _ibv_pd_ex *pd_ex = container_of(pd, struct _ibv_pd_ex, pd); struct _ibv_ah_ex *ah_ex; - pd_ex = container_of(pd, struct _ibv_pd_ex, pd); + ah_ex = calloc(1, sizeof *ah_ex); + if (!ah_ex) + return NULL; + + ah_ex->real_ah = ibv_create_ah(pd_ex->real_pd, attr); + if (!ah_ex->real_ah) { + free(ah_ex); + return NULL; + } + + return &ah_ex->ah; } static int _ibv_destroy_ah_ex(struct ibv_ah *ah) { - struct _ibv_ah_ex *ah_ex; - ah_ex = container_of(ah, struct _ibv_ah_ex, ah); + struct _ibv_ah_ex *ah_ex = container_of(ah, struct _ibv_ah_ex, ah); + int ret; + ret = ibv_destroy_ah(ah_ex->real_ah); + if (ret) + return ret; + + free(ah_ex); + return 0; } static int _ibv_attach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) { - struct _ibv_qp_ex *qp_ex; - qp_ex = container_of(qp, struct _ibv_qp_ex, qp); - + struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + return ibv_attach_mcast(qp_ex->real_qp, gid, lid); } static int _ibv_detach_mcast_ex(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) { - struct _ibv_qp_ex *qp_ex; - qp_ex = container_of(qp, struct _ibv_qp_ex, qp); - + struct _ibv_qp_ex *qp_ex = container_of(qp, struct _ibv_qp_ex, qp); + return ibv_detach_mcast(qp_ex->real_qp, gid, lid); } -static void _ibv_async_event_ex(struct ibv_async_event *event) +int _ibv_get_async_event_ex(struct ibv_context *context, + struct ibv_async_event *event) { } +void _ibv_ack_async_event_ex(struct ibv_async_event *event) +{ + ibv_ack_async_event_ex() +} + struct ibv_context *_ibv_open_device_ex(struct ibv_device *device) { + //XXX } int _ibv_close_device_ex(struct ibv_context *context) { + //XXX } diff --git a/src/device.c b/src/device.c index 9e43138..5b598fd 100644 --- a/src/device.c +++ b/src/device.c @@ -221,6 +221,7 @@ int __ibv_get_async_event(struct ibv_context *context, struct ibv_async_event *event) { struct ibv_kern_async_event ev; + int cqe = 0, qpe = 0, srqe = 0; if (read(context->async_fd, &ev, sizeof ev) != sizeof ev) return -1; @@ -230,6 +231,7 @@ int __ibv_get_async_event(struct ibv_context *context, switch (event->event_type) { case IBV_EVENT_CQ_ERR: event->element.cq = (void *) (uintptr_t) ev.element; + cqe = 1; break; case IBV_EVENT_QP_FATAL: @@ -241,11 +243,13 @@ int __ibv_get_async_event(struct ibv_context *context, case IBV_EVENT_PATH_MIG_ERR: case IBV_EVENT_QP_LAST_WQE_REACHED: event->element.qp = (void *) (uintptr_t) ev.element; + qpe = 1; break; case IBV_EVENT_SRQ_ERR: case IBV_EVENT_SRQ_LIMIT_REACHED: event->element.srq = (void *) (uintptr_t) ev.element; + srqe = 1; break; default: @@ -256,6 +260,13 @@ int __ibv_get_async_event(struct ibv_context *context, if (context->ops.async_event) context->ops.async_event(event); + if (cqe) + event->element.cq = _ibv_verbs_cq(event->element.cq); + else if (qpe) + event->element.qp = _ibv_verbs_qp(event->element.qp); + else if (srqe) + event->element.srq = _ibv_verbs_srq(event->element.srq); + return 0; } default_symver(__ibv_get_async_event, ibv_get_async_event); @@ -265,7 +276,7 @@ void __ibv_ack_async_event(struct ibv_async_event *event) switch (event->event_type) { case IBV_EVENT_CQ_ERR: { - struct ibv_cq *cq = event->element.cq; + struct ibv_cq *cq = _ibv_real_cq(event->element.cq); pthread_mutex_lock(&cq->mutex); ++cq->async_events_completed; @@ -284,7 +295,7 @@ void __ibv_ack_async_event(struct ibv_async_event *event) case IBV_EVENT_PATH_MIG_ERR: case IBV_EVENT_QP_LAST_WQE_REACHED: { - struct ibv_qp *qp = event->element.qp; + struct ibv_qp *qp = _ibv_real_qp(event->element.qp); pthread_mutex_lock(&qp->mutex); ++qp->events_completed; @@ -297,7 +308,7 @@ void __ibv_ack_async_event(struct ibv_async_event *event) case IBV_EVENT_SRQ_ERR: case IBV_EVENT_SRQ_LIMIT_REACHED: { - struct ibv_srq *srq = event->element.srq; + struct ibv_srq *srq = _ibv_real_srq(event->element.srq); pthread_mutex_lock(&srq->mutex); ++srq->events_completed; diff --git a/src/ibverbs.h b/src/ibverbs.h index fa6cd41..3ad0803 100644 --- a/src/ibverbs.h +++ b/src/ibverbs.h @@ -102,4 +102,11 @@ HIDDEN int ibverbs_init(struct ibv_device ***list); (cmd)->response = (uintptr_t) (out); \ } while (0) +struct ibv_srq *_ibv_real_srq(struct ibv_srq *srq); +struct ibv_qp *_ibv_real_qp(struct ibv_qp *qp); +struct ibv_cq *_ibv_real_cq(struct ibv_cq *cq); +struct ibv_srq *_ibv_verbs_srq(struct ibv_srq *srq); +struct ibv_qp *_ibv_verbs_qp(struct ibv_qp *qp); +struct ibv_cq *_ibv_verbs_cq(struct ibv_cq *cq); + #endif /* IB_VERBS_H */ diff --git a/src/verbs.c b/src/verbs.c index b5938f2..7973afc 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -334,18 +334,21 @@ int __ibv_get_cq_event(struct ibv_comp_channel *channel, if (read(channel->fd, &ev, sizeof ev) != sizeof ev) return -1; - *cq = (struct ibv_cq *) (uintptr_t) ev.cq_handle; - *cq_context = (*cq)->cq_context; - + *cq = (struct ibv_cq *) (uintptr_t) ev.cq_handle; if ((*cq)->context->ops.cq_event) (*cq)->context->ops.cq_event(*cq); + if (!verbs_get_ctx((*cq)->context)) + *cq = (struct ibv_cq *) (*cq)->cq_context; + *cq_context = (*cq)->cq_context; + return 0; } default_symver(__ibv_get_cq_event, ibv_get_cq_event); void __ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents) { + cq = _ibv_real_cq(cq); pthread_mutex_lock(&cq->mutex); cq->comp_events_completed += nevents; pthread_cond_signal(&cq->cond); -- 2.46.0