Add ibv_open_qp()

XRC receive QPs are shareable across multiple processes.  Allow
any process with access to the xrc domain to open an existing
QP.  After opening the QP, the process will receive events
related to the QP and be able to modify the QP.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 9aea854e60e995df9d5a81b36a65731d51612ff8..7fad2a2779685eb3d4a688f6988b93872910a26b 100644 (file)
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -122,6 +122,10 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
                      struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
                      struct ibv_create_qp *cmd, size_t cmd_size,
                      struct ibv_create_qp_resp *resp, size_t resp_size);
+int ibv_cmd_open_qp(struct ibv_xrcd *xrcd,
+                   struct ibv_qp *qp, struct ibv_qp_open_attr *attr,
+                   struct ibv_open_qp *cmd, size_t cmd_size,
+                   struct ibv_create_qp_resp *resp, size_t resp_size);
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
                     int attr_mask,
                     struct ibv_qp_init_attr *qp_init_attr,

diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 58867c30197f06d2966beac9a5d23ac22fc09915..68c5fd4481161cb6394afa41bc9b267fb10465fb 100644 (file)
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -88,7 +88,8 @@ enum {
        IB_USER_VERBS_CMD_POST_SRQ_RECV,
        IB_USER_VERBS_CMD_OPEN_XRCD,
        IB_USER_VERBS_CMD_CLOSE_XRCD,
-       IB_USER_VERBS_CMD_CREATE_XSRQ
+       IB_USER_VERBS_CMD_CREATE_XSRQ,
+       IB_USER_VERBS_CMD_OPEN_QP
 };
 
 /*
@@ -476,6 +477,20 @@ struct ibv_create_qp {
        __u64 driver_data[0];
 };
 
+struct ibv_open_qp {
+       __u32 command;
+       __u16 in_words;
+       __u16 out_words;
+       __u64 response;
+       __u64 user_handle;
+       __u32 pd_handle;
+       __u32 qpn;
+       __u8  qp_type;
+       __u8  reserved[7];
+       __u64 driver_data[0];
+};
+
+/* also used for open response */
 struct ibv_create_qp_resp {
        __u32 qp_handle;
        __u32 qpn;
@@ -853,6 +868,7 @@ enum {
        IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
        IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
        IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1,
+       IB_USER_VERBS_CMD_OPEN_QP_V2 = -1,
 };
 
 struct ibv_destroy_cq_v1 {

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index acdd466ff5f9732bdc7e8a652fcbb77ce3de5ea9..a499a50c84db286b07ffa0559d779798be3d254a 100644 (file)
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -459,6 +459,12 @@ struct ibv_qp_init_attr {
        } ext;
 };
 
+struct ibv_qp_open_attr {
+       void                   *qp_context;
+       uint32_t                qp_num;
+       enum ibv_qp_type        qp_type;
+};
+
 enum ibv_qp_attr_mask {
        IBV_QP_STATE                    = 1 <<  0,
        IBV_QP_CUR_STATE                = 1 <<  1,
@@ -490,7 +496,8 @@ enum ibv_qp_state {
        IBV_QPS_RTS,
        IBV_QPS_SQD,
        IBV_QPS_SQE,
-       IBV_QPS_ERR
+       IBV_QPS_ERR,
+       IBV_QPS_UNKNOWN
 };
 
 enum ibv_mig_state {
@@ -766,6 +773,8 @@ struct ibv_xrc_ops {
        struct ibv_xrcd *       (*open_xrcd)(struct ibv_context *context,
                                             int fd, int oflags);
        int                     (*close_xrcd)(struct ibv_xrcd *xrcd);
+       struct ibv_qp *         (*open_qp)(struct ibv_xrcd *xrcd,
+                                          struct ibv_qp_open_attr *attr);
 };
 
 struct ibv_context {
@@ -1094,6 +1103,12 @@ static inline int ibv_post_srq_recv(struct ibv_srq *srq,
 struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
                             struct ibv_qp_init_attr *qp_init_attr);
 
+/**
+ * ibv_open_qp - Open a shareable queue pair.
+ */
+struct ibv_qp *ibv_open_qp(struct ibv_xrcd *xrcd,
+                          struct ibv_qp_open_attr *qp_open_attr);
+
 /**
  * ibv_modify_qp - Modify a queue pair.
  */

diff --git a/src/cmd.c b/src/cmd.c
index cd915177f11256a4116d2ba99069e859dc61c100..3f348c1586d4fc5a0a4defa7c88a2b5b1cd7aa2f 100644 (file)
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -737,6 +737,28 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
        return 0;
 }
 
+int ibv_cmd_open_qp(struct ibv_xrcd *xrcd, struct ibv_qp *qp,
+                   struct ibv_qp_open_attr *attr,
+                   struct ibv_open_qp *cmd, size_t cmd_size,
+                   struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+       IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size);
+
+       cmd->user_handle = (uintptr_t) qp;
+       cmd->pd_handle   = xrcd->handle;
+       cmd->qpn         = attr->qp_num;
+       cmd->qp_type     = attr->qp_type;
+
+       if (write(xrcd->context->cmd_fd, cmd, cmd_size) != cmd_size)
+               return errno;
+
+       VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+       qp->handle     = resp->qp_handle;
+
+       return 0;
+}
+
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
                     int attr_mask,
                     struct ibv_qp_init_attr *init_attr,

diff --git a/src/libibverbs.map b/src/libibverbs.map
index c2f7bb47f96ef18f5df84846fcf7779966720e50..efd671286f263d447de2cf91af409755d78c2c9a 100644 (file)
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -108,4 +108,6 @@ IBVERBS_1.1 {
                ibv_open_xrcd;
                ibv_close_xrcd;
                ibv_create_xsrq;
+               ibv_cmd_open_qp;
+               ibv_open_qp;
 } IBVERBS_1.0;

diff --git a/src/verbs.c b/src/verbs.c
index 40dbef5416ed2bc163f18fac058ce5877c0f0913..163161037862cd6a76ef3b723795c5d23b5aa2a3 100644 (file)
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -502,6 +502,35 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
 }
 default_symver(__ibv_create_qp, ibv_create_qp);
 
+struct ibv_qp *__ibv_open_qp(struct ibv_xrcd *xrcd,
+                            struct ibv_qp_open_attr *qp_open_attr)
+{
+       struct ibv_xrc_ops *ops;
+       struct ibv_qp *qp;
+
+       ops = ibv_get_ext_ops(xrcd->context, IBV_XRC_OPS);
+       qp = ops->open_qp(xrcd, qp_open_attr);
+       if (qp) {
+               qp->context    = xrcd->context;
+               qp->qp_context = qp_open_attr->qp_context;
+
+               qp->pd = NULL;
+               qp->send_cq = qp->recv_cq = NULL;
+               qp->srq = NULL;
+               qp->ext.xrc_recv.xrcd = xrcd;
+
+               qp->qp_num  = qp_open_attr->qp_num;
+               qp->qp_type = qp_open_attr->qp_type;
+               qp->state   = IBV_QPS_UNKNOWN;
+               qp->events_completed = 0;
+               pthread_mutex_init(&qp->mutex, NULL);
+               pthread_cond_init(&qp->cond, NULL);
+       }
+
+       return qp;
+}
+default_symver(__ibv_open_qp, ibv_open_qp);
+
 int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
                   int attr_mask,
                   struct ibv_qp_init_attr *init_attr)