libibverbs: Add ibv_open_qp

XRC receive QPs are shareable across multiple processes.  Allow
any process with access to the xrc domain to open an existing
QP.  After opening the QP, the process will receive events
related to the QP and be able to modify the QP.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index c86109df3b61bdc32a76d890cbbfb7aa64ee4beb..968888fd9008afab7937bbc7b68c7c8566daac2d 100644 (file)
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -126,6 +126,10 @@ int ibv_cmd_create_qp_ex(struct ibv_pd *pd,
                         struct ibv_qp *qp, struct ibv_qp_init_attr_ex *attr_ex,
                         struct ibv_create_qp *cmd, size_t cmd_size,
                         struct ibv_create_qp_resp *resp, size_t resp_size);
+int ibv_cmd_open_qp(struct ibv_xrcd *xrcd,
+                   struct ibv_qp *qp, struct ibv_qp_open_attr *attr,
+                   struct ibv_open_qp *cmd, size_t cmd_size,
+                   struct ibv_create_qp_resp *resp, size_t resp_size);
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
                     int attr_mask,
                     struct ibv_qp_init_attr *qp_init_attr,

diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index b6d5ce9f776f3830c17a17c4bb05ef46636ca31d..e24fa4f912a8cad24df55104443eb745d550e919 100644 (file)
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -88,7 +88,8 @@ enum {
        IB_USER_VERBS_CMD_POST_SRQ_RECV,
        IB_USER_VERBS_CMD_OPEN_XRCD,
        IB_USER_VERBS_CMD_CLOSE_XRCD,
-       IB_USER_VERBS_CMD_CREATE_XSRQ
+       IB_USER_VERBS_CMD_CREATE_XSRQ,
+       IB_USER_VERBS_CMD_OPEN_QP
 };
 
 /*
@@ -476,6 +477,20 @@ struct ibv_create_qp {
        __u64 driver_data[0];
 };
 
+struct ibv_open_qp {
+       __u32 command;
+       __u16 in_words;
+       __u16 out_words;
+       __u64 response;
+       __u64 user_handle;
+       __u32 pd_handle;
+       __u32 qpn;
+       __u8  qp_type;
+       __u8  reserved[7];
+       __u64 driver_data[0];
+};
+
+/* also used for open response */
 struct ibv_create_qp_resp {
        __u32 qp_handle;
        __u32 qpn;
@@ -852,7 +867,8 @@ enum {
        IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
        IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
        IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
-       IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1
+       IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1,
+       IB_USER_VERBS_CMD_OPEN_QP_V2 = -1
 };
 
 struct ibv_destroy_cq_v1 {

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 68ea6579c123369bcaaaf4d5235383d62fe185dc..765ac8ba40453c7795265f07968ab5edf501c548 100644 (file)
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -504,6 +504,20 @@ struct ibv_qp_init_attr_ex {
        struct ibv_xrcd        *xrcd;
 };
 
+enum ibv_qp_open_attr_mask {
+       IBV_QP_OPEN_ATTR_CONTEXT        = 1 << 0,
+       IBV_QP_OPEN_ATTR_NUM            = 1 << 1,
+       IBV_QP_OPEN_ATTR_TYPE           = 1 << 2,
+       IBV_QP_OPEN_ATTR_RESERVED       = 1 << 3
+ };
+
+struct ibv_qp_open_attr {
+       uint32_t                comp_mask;
+       uint32_t                qp_num;
+       void                   *qp_context;
+       enum ibv_qp_type        qp_type;
+};
+
 enum ibv_qp_attr_mask {
        IBV_QP_STATE                    = 1 <<  0,
        IBV_QP_CUR_STATE                = 1 <<  1,
@@ -535,7 +549,8 @@ enum ibv_qp_state {
        IBV_QPS_RTS,
        IBV_QPS_SQD,
        IBV_QPS_SQE,
-       IBV_QPS_ERR
+       IBV_QPS_ERR,
+       IBV_QPS_UNKNOWN
 };
 
 enum ibv_mig_state {
@@ -848,6 +863,8 @@ struct verbs_context {
        int (*drv_new_func1) ();        new corresponding provider call of func1
        int (*lib_new_func1) ();        New library call func1
        */
+       struct ibv_qp *         (*open_qp)(struct ibv_xrcd *xrcd,
+                                          struct ibv_qp_open_attr *attr);
        struct ibv_qp *         (*create_qp_ex)(struct ibv_pd *pd,
                                                struct ibv_qp_init_attr_ex *qp_init_attr_ex);
        struct ibv_srq *        (*create_srq_ex)(struct ibv_pd *pd,
@@ -1170,6 +1187,12 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
 struct ibv_qp *ibv_create_qp_ex(struct ibv_pd *pd,
                                struct ibv_qp_init_attr_ex *qp_init_attr_ex);
 
+/**
+ * ibv_open_qp - Open a shareable queue pair.
+ */
+struct ibv_qp *ibv_open_qp(struct ibv_xrcd *xrcd,
+                          struct ibv_qp_open_attr *qp_open_attr);
+
 /**
  * ibv_modify_qp - Modify a queue pair.
  */

diff --git a/src/cmd.c b/src/cmd.c
index b9d96f9095b70c6d985e0c56aaf3fc44ed5e9f61..d0950daee913a4042429df4674fdd0465e75bb22 100644 (file)
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -716,6 +716,28 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
        return ret;
 }
 
+int ibv_cmd_open_qp(struct ibv_xrcd *xrcd, struct ibv_qp *qp,
+                   struct ibv_qp_open_attr *attr,
+                   struct ibv_open_qp *cmd, size_t cmd_size,
+                   struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+       IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size);
+
+       cmd->user_handle = (uintptr_t) qp;
+       cmd->pd_handle   = xrcd->handle;
+       cmd->qpn         = attr->qp_num;
+       cmd->qp_type     = attr->qp_type;
+
+       if (write(xrcd->context->cmd_fd, cmd, cmd_size) != cmd_size)
+               return errno;
+
+       VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+       qp->handle     = resp->qp_handle;
+
+       return 0;
+}
+
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
                     int attr_mask,
                     struct ibv_qp_init_attr *init_attr,

diff --git a/src/libibverbs.map b/src/libibverbs.map
index 9ee1cf21ae8f5577e833b24438c5add4ac4f9607..93903c2b6f7e51fe23e3bab730b21d33fe2a59c5 100644 (file)
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -106,5 +106,7 @@ IBVERBS_1.1 {
                ibv_cmd_create_srq_ex;
                ibv_create_qp_ex;
                ibv_cmd_create_qp_ex;
+               ibv_open_qp;
+               ibv_cmd_open_qp;
                
 } IBVERBS_1.0;

diff --git a/src/verbs.c b/src/verbs.c
index 9c6391715627e3cf397970fcb504d3da4fc71801..7d55127fdd9f2925418ecf0f1ae7e99914e40979 100644 (file)
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -527,6 +527,42 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
 }
 default_symver(__ibv_create_qp, ibv_create_qp);
 
+struct ibv_qp *__ibv_open_qp(struct ibv_xrcd *xrcd,
+                            struct ibv_qp_open_attr *qp_open_attr)
+{
+       struct verbs_context *context_ex;
+       struct ibv_qp *qp;
+
+       context_ex = verbs_get_ctx(xrcd->context);
+       if (!context_ex->open_qp ||
+           qp_open_attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED) {
+               errno = ENOSYS;
+               return NULL;
+       }
+
+       qp = context_ex->open_qp(xrcd, qp_open_attr);
+       if (qp) {
+               qp->context    = xrcd->context;
+               qp->qp_context = qp_open_attr->qp_context;
+
+               qp->pd = NULL;
+               qp->send_cq = qp->recv_cq = NULL;
+               qp->srq = NULL;
+               qp->comp_mask |= IBV_QP_XRCD;
+               qp->xrcd = xrcd;
+
+               qp->qp_num  = qp_open_attr->qp_num;
+               qp->qp_type = qp_open_attr->qp_type;
+               qp->state   = IBV_QPS_UNKNOWN;
+               qp->events_completed = 0;
+               pthread_mutex_init(&qp->mutex, NULL);
+               pthread_cond_init(&qp->cond, NULL);
+       }
+
+       return qp;
+}
+default_symver(__ibv_open_qp, ibv_open_qp);
+
 int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
                   int attr_mask,
                   struct ibv_qp_init_attr *init_attr)