From 1c951c55f9fecd1b36bed54ea8f781f2c41cb5b4 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Mon, 17 Sep 2012 21:10:34 -0700 Subject: [PATCH] Refresh of xrc_qp --- include/infiniband/driver.h | 4 +++ include/infiniband/verbs.h | 11 +++++--- src/cmd.c | 45 ++++++++++++++++++++++++------- src/libibverbs.map | 2 ++ src/verbs.c | 54 ++++++++++++++++++++++++++++++------- 5 files changed, 94 insertions(+), 22 deletions(-) diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h index cac48ab..c86109d 100644 --- a/include/infiniband/driver.h +++ b/include/infiniband/driver.h @@ -122,6 +122,10 @@ int ibv_cmd_create_qp(struct ibv_pd *pd, struct ibv_qp *qp, struct ibv_qp_init_attr *attr, struct ibv_create_qp *cmd, size_t cmd_size, struct ibv_create_qp_resp *resp, size_t resp_size); +int ibv_cmd_create_qp_ex(struct ibv_pd *pd, + struct ibv_qp *qp, struct ibv_qp_init_attr_ex *attr_ex, + struct ibv_create_qp *cmd, size_t cmd_size, + struct ibv_create_qp_resp *resp, size_t resp_size); int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr, int attr_mask, struct ibv_qp_init_attr *qp_init_attr, diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h index 1d03c4b..68ea657 100644 --- a/include/infiniband/verbs.h +++ b/include/infiniband/verbs.h @@ -106,7 +106,8 @@ enum ibv_device_cap_flags { IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11, IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12, IBV_DEVICE_SRQ_RESIZE = 1 << 13, - IBV_DEVICE_N_NOTIFY_CQ = 1 << 14 + IBV_DEVICE_N_NOTIFY_CQ = 1 << 14, + IBV_DEVICE_XRC = 1 << 20 }; enum ibv_atomic_cap { @@ -667,8 +668,6 @@ struct ibv_srq { uint32_t srq_num; }; -// set device xrc support -//XXX TO DO: check mask for all calls enum ibv_qp_mask { IBV_QP_XRCD = 1 << 0, IBV_QP_RESERVED = 1 << 1 @@ -691,6 +690,7 @@ struct ibv_qp { uint32_t events_completed; uint32_t comp_mask; + struct ibv_xrcd *xrcd; }; enum ibv_comp_channel_mask { @@ -848,6 +848,8 @@ struct verbs_context { int (*drv_new_func1) (); new corresponding provider call of func1 int (*lib_new_func1) (); New library call func1 */ + struct ibv_qp * (*create_qp_ex)(struct ibv_pd *pd, + struct ibv_qp_init_attr_ex *qp_init_attr_ex); struct ibv_srq * (*create_srq_ex)(struct ibv_pd *pd, struct ibv_srq_init_attr_ex *srq_init_attr_ex); struct ibv_xrcd * (*open_xrcd)(struct ibv_context *context, @@ -1165,6 +1167,9 @@ static inline int ibv_post_srq_recv(struct ibv_srq *srq, struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr); +struct ibv_qp *ibv_create_qp_ex(struct ibv_pd *pd, + struct ibv_qp_init_attr_ex *qp_init_attr_ex); + /** * ibv_modify_qp - Modify a queue pair. */ diff --git a/src/cmd.c b/src/cmd.c index 1c4bfb3..e9f9727 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -628,18 +628,31 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq) return 0; } -int ibv_cmd_create_qp(struct ibv_pd *pd, - struct ibv_qp *qp, struct ibv_qp_init_attr *attr, - struct ibv_create_qp *cmd, size_t cmd_size, - struct ibv_create_qp_resp *resp, size_t resp_size) +int ibv_cmd_create_qp_ex(struct ibv_pd *pd, + struct ibv_qp *qp, struct ibv_qp_init_attr_ex *attr_ex, + struct ibv_create_qp *cmd, size_t cmd_size, + struct ibv_create_qp_resp *resp, size_t resp_size) { + struct ibv_context *context; + IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size); cmd->user_handle = (uintptr_t) qp; - cmd->pd_handle = pd->handle; - cmd->send_cq_handle = attr->send_cq->handle; - cmd->recv_cq_handle = attr->recv_cq->handle; - cmd->srq_handle = attr->srq ? attr->srq->handle : 0; + + if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_XRCD) { + context = attr_ex->xrcd->context; + cmd->pd_handle = attr_ex->xrcd->handle; + } else { + context = pd->context; + cmd->pd_handle = pd->handle; + cmd->send_cq_handle = attr->send_cq->handle; + + if (attr->qp_type != IBV_QPT_XRC_SEND) { + cmd->recv_cq_handle = attr_ex->recv_cq->handle; + cmd->srq_handle = attr_ex->srq ? attr->srq->handle : 0; + } + } + cmd->max_send_wr = attr->cap.max_send_wr; cmd->max_recv_wr = attr->cap.max_recv_wr; cmd->max_send_sge = attr->cap.max_send_sge; @@ -650,14 +663,14 @@ int ibv_cmd_create_qp(struct ibv_pd *pd, cmd->is_srq = !!attr->srq; cmd->reserved = 0; - if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) + if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) return errno; (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); qp->handle = resp->qp_handle; qp->qp_num = resp->qpn; - qp->context = pd->context; + qp->context = context; if (abi_ver > 3) { attr->cap.max_recv_sge = resp->max_recv_sge; @@ -686,6 +699,18 @@ int ibv_cmd_create_qp(struct ibv_pd *pd, return 0; } +int ibv_cmd_create_qp(struct ibv_pd *pd, + struct ibv_qp *qp, struct ibv_qp_init_attr *attr, + struct ibv_create_qp *cmd, size_t cmd_size, + struct ibv_create_qp_resp *resp, size_t resp_size) +{ + struct ibv_qp_init_attr_ex attr_ex; + + memcpy(&attr_ex, attr, sizeof attr); + attr_ex.comp_mask = 0; + return ibv_cmd_create_qp_ex(pd, qp, &attr_ex, cmd, cmd_size, resp, resp_size); +} + int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr, diff --git a/src/libibverbs.map b/src/libibverbs.map index 92bfabd..9ee1cf2 100644 --- a/src/libibverbs.map +++ b/src/libibverbs.map @@ -104,5 +104,7 @@ IBVERBS_1.1 { ibv_cmd_close_xrcd; ibv_create_srq_ex; ibv_cmd_create_srq_ex; + ibv_create_qp_ex; + ibv_cmd_create_qp_ex; } IBVERBS_1.0; diff --git a/src/verbs.c b/src/verbs.c index d08ee79..d9cd8d7 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -462,19 +462,44 @@ int __ibv_destroy_srq(struct ibv_srq *srq) } default_symver(__ibv_destroy_srq, ibv_destroy_srq); -struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd, - struct ibv_qp_init_attr *qp_init_attr) +struct ibv_qp *__ibv_create_qp_ex(struct ibv_pd *pd, + struct ibv_qp_init_attr_ex *qp_init_attr_ex) { - struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr); + struct verbs_context *context_ex = NULL; + struct ibv_context *context; + struct ibv_qp *qp; + + if (pd) { + context = pd->context; + } else { + if (qp_init_attr_ex->comp_mask & IBV_QP_INIT_ATTR_XRCD) + context = qp_init_attr_ex->xrcd->context; + } + if (!context) { + errno = EINVAL; + return NULL + } + + context_ex = verbs_get_ctx(context); + if (qp_init_attr_ex->comp_mask) { + if (!context_ex->create_qp_ex || + qp_init_attr_ex->comp_mask >= IBV_QP_INIT_ATTR_RESERVED) { + errno = ENOSYS; + return NULL; + } + qp = context_ex->create_qp_ex(pd, qp_init_attr_ex); + } else { + qp = context->ops.create_qp(pd, (struct ibv_qp_init_attr *) qp_init_attr_ex); + } if (qp) { - qp->context = pd->context; - qp->qp_context = qp_init_attr->qp_context; + qp->context = context; + qp->qp_context = qp_init_attr_ex->qp_context; qp->pd = pd; - qp->send_cq = qp_init_attr->send_cq; - qp->recv_cq = qp_init_attr->recv_cq; - qp->srq = qp_init_attr->srq; - qp->qp_type = qp_init_attr->qp_type; + qp->send_cq = qp_init_attr_ex->send_cq; + qp->recv_cq = qp_init_attr_ex->recv_cq; + qp->srq = qp_init_attr_ex->srq; + qp->qp_type = qp_init_attr_ex->qp_type; qp->state = IBV_QPS_RESET; qp->events_completed = 0; pthread_mutex_init(&qp->mutex, NULL); @@ -483,6 +508,17 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd, return qp; } +default_symver(__ibv_create_qp_ex, ibv_create_qp_ex); + +struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd, + struct ibv_qp_init_attr *qp_init_attr) +{ + struct ibv_qp_init_attr_ex qp_init_attr_ex; + + memcpy(&qp_init_attr_ex, qp_init_attr, sizeof qp_init_attr); + qp_init_attr_ex.comp_mask = 0; + return ibv_create_qp_ex(pd, &qp_init_attr_ex); +} default_symver(__ibv_create_qp, ibv_create_qp); int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, -- 2.46.0