libibverbs: Introduce XRC domains

XRC introduces several new concepts and structures, one of
which is the XRC domain.

XRC domains: xrcd's are a type of protection domain used to
associate shared receive queues with xrc queue pairs.  Since
xrcd are meant to be shared among multiple processes, we
introduce new APIs to open/close xrcd's.

The user to kernel ABI is extended to account for opening/
closing the xrcd.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index f22f287dad1dd3eb2a17a5a8deb7e15adce06c5f..ce88442530f769a5dce2cda0f31e0de6bf0c3650 100644 (file)
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -53,6 +53,17 @@
  */
 #define IBV_DEVICE_LIBRARY_EXTENSION rdmav2
 
+enum verbs_xrcd_mask {
+       VERBS_XRCD_HANDLE       = 1 << 0,
+       VERBS_XRCD_RESERVED     = 1 << 1
+};
+
+struct verbs_xrcd {
+       struct ibv_xrcd         xrcd;
+       uint64_t                comp_mask;
+       uint32_t                handle;
+};
+
 typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
                                                   int abi_version);
 typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
@@ -78,6 +89,11 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
                     struct ibv_alloc_pd *cmd, size_t cmd_size,
                     struct ibv_alloc_pd_resp *resp, size_t resp_size);
 int ibv_cmd_dealloc_pd(struct ibv_pd *pd);
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
+                     struct ibv_xrcd_init_attr *attr,
+                     struct ibv_open_xrcd *cmd, size_t cmd_size,
+                     struct ibv_open_xrcd_resp *resp, size_t resp_size);
+int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd);
 #define IBV_CMD_REG_MR_HAS_RESP_PARAMS
 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
                   uint64_t hca_va, int access,

diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index e7f898106d52a471463a270b4c8a0ca7d48b5031..6f40ad2161535e83980e1328a9b802a862ca30d3 100644 (file)
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -85,7 +85,9 @@ enum {
        IB_USER_VERBS_CMD_MODIFY_SRQ,
        IB_USER_VERBS_CMD_QUERY_SRQ,
        IB_USER_VERBS_CMD_DESTROY_SRQ,
-       IB_USER_VERBS_CMD_POST_SRQ_RECV
+       IB_USER_VERBS_CMD_POST_SRQ_RECV,
+       IB_USER_VERBS_CMD_OPEN_XRCD,
+       IB_USER_VERBS_CMD_CLOSE_XRCD,
 };
 
 /*
@@ -246,6 +248,27 @@ struct ibv_dealloc_pd {
        __u32 pd_handle;
 };
 
+struct ibv_open_xrcd {
+       __u32 command;
+       __u16 in_words;
+       __u16 out_words;
+       __u64 response;
+       __u32 fd;
+       __u32 oflags;
+       __u64 driver_data[0];
+};
+
+struct ibv_open_xrcd_resp {
+       __u32 xrcd_handle;
+};
+
+struct ibv_close_xrcd {
+       __u32 command;
+       __u16 in_words;
+       __u16 out_words;
+       __u32 xrcd_handle;
+};
+
 struct ibv_reg_mr {
        __u32 command;
        __u16 in_words;
@@ -804,6 +827,8 @@ enum {
         * trick opcodes in IBV_INIT_CMD() doesn't break.
         */
        IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
+       IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
+       IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
 };
 
 struct ibv_modify_srq_v3 {

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index e4361aae9063fa4ad0d51982ebb0c87da0a20175..cf8299a5f470b28c2d5d4be2fd742900f4096a42 100644 (file)
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
- * Copyright (c) 2004 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2004, 2011-2012 Intel Corporation.  All rights reserved.
  * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
  *
@@ -39,6 +39,7 @@
 #include <stdint.h>
 #include <pthread.h>
 #include <stddef.h>
+#include <errno.h>
 
 #ifdef __cplusplus
 #  define BEGIN_C_DECLS extern "C" {
@@ -312,6 +313,22 @@ struct ibv_pd {
        uint32_t                handle;
 };
 
+enum ibv_xrcd_init_attr_mask {
+       IBV_XRCD_INIT_ATTR_FD       = 1 << 0,
+       IBV_XRCD_INIT_ATTR_OFLAGS   = 1 << 1,
+       IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2
+};
+
+struct ibv_xrcd_init_attr {
+       uint32_t comp_mask;
+       int      fd;
+       int      oflags;
+};
+
+struct ibv_xrcd {
+       struct ibv_context     *context;
+};
+
 enum ibv_rereg_mr_flags {
        IBV_REREG_MR_CHANGE_TRANSLATION = (1 << 0),
        IBV_REREG_MR_CHANGE_PD          = (1 << 1),
@@ -728,11 +745,15 @@ struct ibv_context {
 };
 
 enum verbs_context_mask {
-       VERBS_CONTEXT_RESERVED = 1 << 0
+       VERBS_CONTEXT_XRCD      = 1 << 0,
+       VERBS_CONTEXT_RESERVED  = 1 << 1
 };
 
 struct verbs_context {
        /*  "grows up" - new fields go here */
+       struct ibv_xrcd *       (*open_xrcd)(struct ibv_context *context,
+                                            struct ibv_xrcd_init_attr *xrcd_init_attr);
+       int                     (*close_xrcd)(struct ibv_xrcd *xrcd);
        uint64_t has_comp_mask;
        size_t   sz;    /* Must be immediately before struct ibv_context */
        struct ibv_context context;/* Must be last field in the struct */
@@ -835,7 +856,7 @@ static inline int ___ibv_query_port(struct ibv_context *context,
                                    uint8_t port_num,
                                    struct ibv_port_attr *port_attr)
 {
-       /* For compatability when running with old libibverbs */
+       /* For compatibility when running with old libibverbs */
        port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
        port_attr->reserved   = 0;
 
@@ -867,6 +888,29 @@ struct ibv_pd *ibv_alloc_pd(struct ibv_context *context);
  */
 int ibv_dealloc_pd(struct ibv_pd *pd);
 
+/**
+ * ibv_open_xrcd - Open an extended connection domain
+ */
+static inline struct ibv_xrcd *
+ibv_open_xrcd(struct ibv_context *context, struct ibv_xrcd_init_attr *xrcd_init_attr)
+{
+       struct verbs_context *vctx = verbs_get_ctx_op(context, open_xrcd);
+       if (!vctx) {
+               errno = ENOSYS;
+               return NULL;
+       }
+       return vctx->open_xrcd(context, xrcd_init_attr);
+}
+
+/**
+ * ibv_close_xrcd - Close an extended connection domain
+ */
+static inline int ibv_close_xrcd(struct ibv_xrcd *xrcd)
+{
+       struct verbs_context *vctx = verbs_get_ctx(xrcd->context);
+       return vctx->close_xrcd(xrcd);
+}
+
 /**
  * ibv_reg_mr - Register a memory region
  */

diff --git a/src/cmd.c b/src/cmd.c
index 86350fd816c94502e623086b4377b42451ef6829..fe877239472a3ce4e07f1c19bee7ddb37a161ad1 100644 (file)
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -194,6 +194,47 @@ int ibv_cmd_dealloc_pd(struct ibv_pd *pd)
        return 0;
 }
 
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
+                     struct ibv_xrcd_init_attr *attr,
+                     struct ibv_open_xrcd *cmd, size_t cmd_size,
+                     struct ibv_open_xrcd_resp *resp, size_t resp_size)
+{
+       IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size);
+
+       if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED)
+               return ENOSYS;
+
+       if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) ||
+           !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS))
+               return EINVAL;
+
+       cmd->fd = attr->fd;
+       cmd->oflags = attr->oflags;
+       if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+               return errno;
+
+       VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+       xrcd->xrcd.context = context;
+       xrcd->comp_mask = VERBS_XRCD_HANDLE;
+       xrcd->handle  = resp->xrcd_handle;
+
+       return 0;
+}
+
+int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd)
+{
+       struct ibv_close_xrcd cmd;
+
+       IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD);
+       cmd.xrcd_handle = xrcd->handle;
+
+       if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
+               return errno;
+
+       return 0;
+}
+
 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
                   uint64_t hca_va, int access,
                   struct ibv_mr *mr, struct ibv_reg_mr *cmd,

diff --git a/src/libibverbs.map b/src/libibverbs.map
index ee9adea7105d2fb268d6fbf7b136a1ec21b3bc01..9a15f3fb80945215b40c1c865d9a35490d54aa66 100644 (file)
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -97,4 +97,8 @@ IBVERBS_1.1 {
                ibv_port_state_str;
                ibv_event_type_str;
                ibv_wc_status_str;
+               
+               ibv_cmd_open_xrcd;
+               ibv_cmd_close_xrcd;
+               
 } IBVERBS_1.0;