Define a common libibverbs extension to support XRC.
XRC introduces several new concepts and structures:
XRC domains: xrcd's are a type of protection domain used to
associate shared receive queues with xrc queue pairs. Since
xrcd are meant to be shared among multiple processes, we
introduce new APIs to open/close xrcd's.
XRC shared receive queues: xrc srq's are similar to normal
srq's, except that they are bound to an xrcd, rather
than to a protection domain. Based on the current spec
and implementation, they are only usable with xrc qps. To
support xrc srq's, we extend the existing srq_init_attr
structure to include an srq type and other needed information.
The extended fields are ignored unless extensions are being
used to support existing applications.
XRC queue pairs: xrc defines two new types of QPs. The
initiator, or send-side, xrc qp behaves similar to a send-
only RC qp. xrc send qp's are managed through the existing
QP functions. The send_wr structure is extended in a back-
wards compatible way to support posting sends on a send xrc
qp, which require specifying the remote xrc srq.
The target, or receive-side, xrc qp behaves differently
than other implemented qp's. A recv xrc qp can be created,
modified, and destroyed like other qp's through the existing
calls. The qp_init_attr structure is extended for xrc qp's,
with extension support dependent upon the qp_type being
defined correctly.
Because xrc recv qp's are bound to an xrcd, rather than a pd,
it is intended to be used among multiple processes. Any process
with access to an xrcd may allocate and connect an xrc recv qp.
The actual xrc recv qp is allocated and managed by the kernel.
If the owning process explicit destroys the xrc recv qp, it is
destroyed. However, if the xrc recv qp is left open when the
user process exits or closes its device, then the lifetime of
the xrc recv qp is bound with the lifetime of the xrcd.
The user to kernel ABI is extended to account for opening/
closing the xrcd and the creation of the extended srq type.
Signed-off-by: Sean Hefty <sean.hefty@intel.com>
struct ibv_alloc_pd *cmd, size_t cmd_size,
struct ibv_alloc_pd_resp *resp, size_t resp_size);
int ibv_cmd_dealloc_pd(struct ibv_pd *pd);
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct ibv_xrcd *xrcd,
+ int fd, int oflags,
+ struct ibv_open_xrcd *cmd, size_t cmd_size,
+ struct ibv_open_xrcd_resp *resp, size_t resp_size);
+int ibv_cmd_close_xrcd(struct ibv_xrcd *xrcd);
#define IBV_CMD_REG_MR_HAS_RESP_PARAMS
int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
uint64_t hca_va, int access,
struct ibv_srq *srq, struct ibv_srq_init_attr *attr,
struct ibv_create_srq *cmd, size_t cmd_size,
struct ibv_create_srq_resp *resp, size_t resp_size);
+int ibv_cmd_create_srq_ex(struct ibv_pd *pd,
+ struct ibv_srq *srq, struct ibv_srq_init_attr_ex *attr_ex,
+ struct ibv_create_srq_ex *cmd, size_t cmd_size,
+ struct ibv_create_srq_resp *resp, size_t resp_size);
int ibv_cmd_modify_srq(struct ibv_srq *srq,
struct ibv_srq_attr *srq_attr,
int srq_attr_mask,
IB_USER_VERBS_CMD_MODIFY_SRQ,
IB_USER_VERBS_CMD_QUERY_SRQ,
IB_USER_VERBS_CMD_DESTROY_SRQ,
- IB_USER_VERBS_CMD_POST_SRQ_RECV
+ IB_USER_VERBS_CMD_POST_SRQ_RECV,
+ IB_USER_VERBS_CMD_OPEN_XRCD,
+ IB_USER_VERBS_CMD_CLOSE_XRCD,
+ IB_USER_VERBS_CMD_CREATE_XSRQ
};
/*
__u32 pd_handle;
};
+struct ibv_open_xrcd {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u32 fd;
+ __u32 oflags;
+ __u64 driver_data[0];
+};
+
+struct ibv_open_xrcd_resp {
+ __u32 xrcd_handle;
+};
+
+struct ibv_close_xrcd {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u32 xrcd_handle;
+};
+
struct ibv_reg_mr {
__u32 command;
__u16 in_words;
__u32 remote_qkey;
__u32 reserved;
} ud;
+ struct {
+ __u64 reserved[3];
+ __u32 reserved2;
+ __u32 remote_srqn;
+ } xrc;
} wr;
};
__u64 driver_data[0];
};
+struct ibv_create_xsrq {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u64 user_handle;
+ __u32 srq_type;
+ __u32 pd_handle;
+ __u32 max_wr;
+ __u32 max_sge;
+ __u32 srq_limit;
+ __u32 reserved;
+ __u32 xrcd_handle;
+ __u32 cq_handle;
+ __u64 driver_data[0];
+};
+
struct ibv_create_srq_resp {
__u32 srq_handle;
__u32 max_wr;
__u32 max_sge;
- __u32 reserved;
+ __u32 srqn;
};
struct ibv_modify_srq {
* trick opcodes in IBV_INIT_CMD() doesn't break.
*/
IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
+ IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
+ IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
+ IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1,
};
struct ibv_destroy_cq_v1 {
/*
* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2004 Intel Corporation. All rights reserved.
+ * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved.
* Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2005 PathScale, Inc. All rights reserved.
*
IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11,
IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
IBV_DEVICE_SRQ_RESIZE = 1 << 13,
- IBV_DEVICE_N_NOTIFY_CQ = 1 << 14
+ IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
+ IBV_DEVICE_XRC = 1 << 20
};
enum ibv_atomic_cap {
uint32_t handle;
};
+enum ibv_xrcd_mask {
+ IBV_XRCD_CONTEXT = 1 << 0,
+ IBV_XRCD_HANDLE = 1 << 1,
+ IBV_XRCD_RESERVED = 1 << 2
+};
+
+struct ibv_xrcd {
+ uint64_t comp_mask;
+ struct ibv_context *context;
+ uint32_t handle;
+};
+
enum ibv_rereg_mr_flags {
IBV_REREG_MR_CHANGE_TRANSLATION = (1 << 0),
IBV_REREG_MR_CHANGE_PD = (1 << 1),
struct ibv_srq_attr attr;
};
+enum ibv_srq_type {
+ IBV_SRQT_BASIC,
+ IBV_SRQT_XRC
+};
+
+enum ibv_srq_init_attr_mask {
+ IBV_SRQ_INIT_ATTR_SRQ_TYPE = 1 << 0,
+ IBV_SRQ_INIT_ATTR_XRCD = 1 << 1,
+ IBV_SRQ_INIT_ATTR_CQ = 1 << 2,
+ IBV_SRQ_INIT_ATTR_RESERVED = 1 << 3
+};
+
+struct ibv_srq_init_attr_ex {
+ void *srq_context;
+ struct ibv_srq_attr attr;
+
+ uint64_t comp_mask;
+ enum ibv_srq_type srq_type;
+ struct ibv_xrcd *xrcd;
+ struct ibv_cq *cq;
+};
+
enum ibv_qp_type {
IBV_QPT_RC = 2,
IBV_QPT_UC,
- IBV_QPT_UD
+ IBV_QPT_UD,
+ IBV_QPT_XRC_SEND = 9,
+ IBV_QPT_XRC_RECV
};
struct ibv_qp_cap {
int sq_sig_all;
};
+enum ibv_qp_init_attr_mask {
+ IBV_QP_INIT_ATTR_XRCD = 1 << 0,
+ IBV_QP_INIT_ATTR_RESERVED = 1 << 1
+};
+
+struct ibv_qp_init_attr_ex {
+ void *qp_context;
+ struct ibv_cq *send_cq;
+ struct ibv_cq *recv_cq;
+ struct ibv_srq *srq;
+ struct ibv_qp_cap cap;
+ enum ibv_qp_type qp_type;
+ int sq_sig_all;
+
+ uint64_t comp_mask;
+ struct ibv_xrcd *xrcd;
+};
+
enum ibv_qp_attr_mask {
IBV_QP_STATE = 1 << 0,
IBV_QP_CUR_STATE = 1 << 1,
uint32_t remote_qpn;
uint32_t remote_qkey;
} ud;
+ struct {
+ uint64_t reserved[3];
+ uint32_t reserved2;
+ uint32_t remote_srqn;
+ } xrc;
} wr;
};
int mw_access_flags;
};
+enum ibv_srq_mask {
+ IBV_SRQ_TYPE = 1 << 0,
+ IBV_SRQ_XRCD = 1 << 1,
+ IBV_SRQ_CQ = 1 << 2,
+ IBV_SRQ_NUM = 1 << 3,
+ IBV_SRQ_RESERVED = 1 << 4
+};
+
struct ibv_srq {
struct ibv_context *context;
void *srq_context;
pthread_mutex_t mutex;
pthread_cond_t cond;
uint32_t events_completed;
+
+ uint64_t comp_mask;
+ enum ibv_srq_type srq_type;
+ struct ibv_xrcd *xrcd;
+ struct ibv_cq *cq;
+ uint32_t srq_num;
+};
+
+enum ibv_qp_mask {
+ IBV_QP_XRCD = 1 << 0,
+ IBV_QP_RESERVED = 1 << 1
};
struct ibv_qp {
pthread_mutex_t mutex;
pthread_cond_t cond;
uint32_t events_completed;
+
+ uint64_t comp_mask;
+ struct ibv_xrcd *xrcd;
};
struct ibv_comp_channel {
};
struct verbs_context {
-
/* "grows up" - new fields go here
int (*drv_new_func1) (); new corresponding provider call of func1
int (*lib_new_func1) (); New library call func1
*/
+ struct ibv_qp * (*create_qp_ex)(struct ibv_pd *pd,
+ struct ibv_qp_init_attr_ex *attr_ex);
+ struct ibv_srq * (*create_srq_ex)(struct ibv_pd *pd,
+ struct ibv_srq_init_attr_ex *srq_init_attr_ex);
+ struct ibv_xrcd * (*open_xrcd)(struct ibv_context *context,
+ int fd, int oflags);
+ int (*close_xrcd)(struct ibv_xrcd *xrcd);
size_t sz; /* Set by library on struct allocation,must be
* located right before struct ibv_context
*/
uint8_t port_num,
struct ibv_port_attr *port_attr)
{
- /* For compatability when running with old libibverbs */
+ /* For compatibility when running with old libibverbs */
port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
port_attr->reserved = 0;
*/
int ibv_dealloc_pd(struct ibv_pd *pd);
+/**
+ * ibv_open_xrcd - Open an extended connection domain
+ */
+struct ibv_xrcd *ibv_open_xrcd(struct ibv_context *context, int fd, int oflags);
+
+/**
+ * ibv_close_xrcd - Close an extended connection domain
+ */
+int ibv_close_xrcd(struct ibv_xrcd *xrcd);
+
/**
* ibv_reg_mr - Register a memory region
*/
struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
struct ibv_srq_init_attr *srq_init_attr);
+struct ibv_srq *ibv_create_srq_ex(struct ibv_pd *pd,
+ struct ibv_srq_init_attr_ex *srq_init_attr_ex);
+
/**
* ibv_modify_srq - Modifies the attributes for the specified SRQ.
* @srq: The SRQ to modify.
struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr);
+struct ibv_qp *ibv_create_qp_ex(struct ibv_pd *pd,
+ struct ibv_qp_init_attr_ex *qp_init_attr_ex);
+
/**
* ibv_modify_qp - Modify a queue pair.
*/
return 0;
}
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct ibv_xrcd *xrcd,
+ int fd, int oflags,
+ struct ibv_open_xrcd *cmd, size_t cmd_size,
+ struct ibv_open_xrcd_resp *resp, size_t resp_size)
+{
+ IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size);
+
+ cmd->fd = fd;
+ cmd->oflags = oflags;
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+ xrcd->handle = resp->xrcd_handle;
+ xrcd->context = context;
+
+ return 0;
+}
+
+int ibv_cmd_close_xrcd(struct ibv_xrcd *xrcd)
+{
+ struct ibv_close_xrcd cmd;
+
+ IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD);
+ cmd.xrcd_handle = xrcd->handle;
+
+ if (write(xrcd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
+ return errno;
+
+ return 0;
+}
+
int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
uint64_t hca_va, int access,
struct ibv_mr *mr, struct ibv_reg_mr *cmd,
return 0;
}
+int ibv_cmd_create_srq_ex(struct ibv_pd *pd,
+ struct ibv_srq *srq, struct ibv_srq_init_attr_ex *attr_ex,
+ struct ibv_create_xsrq *cmd, size_t cmd_size,
+ struct ibv_create_srq_resp *resp, size_t resp_size)
+{
+ IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size);
+ cmd->user_handle = (uintptr_t) srq;
+ cmd->pd_handle = pd->handle;
+ cmd->max_wr = attr_ex->attr.max_wr;
+ cmd->max_sge = attr_ex->attr.max_sge;
+ cmd->srq_limit = attr_ex->attr.srq_limit;
+
+ if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_SRQ_TYPE) {
+ cmd->srq_type = attr_ex->srq_type;
+ switch (attr_ex->srq_type) {
+ case IBV_SRQT_XRC:
+ cmd->xrcd_handle = attr_ex->xrcd->handle;
+ cmd->cq_handle = attr_ex->cq->handle;
+ break;
+ default:
+ break;
+ }
+ } else {
+ cmd->srq_type = IBV_SRQT_BASIC;
+ }
+
+ if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+ srq->handle = resp->srq_handle;
+ srq->context = pd->context;
+ if (cmd->srq_type == IBV_SRQT_XRC) {
+ srq->comp_mask |= IBV_SRQ_NUM;
+ srq->srq_num = resp->srqn;
+ }
+
+ attr_ex->attr.max_wr = resp->max_wr;
+ attr_ex->attr.max_sge = resp->max_sge;
+
+ return 0;
+}
+
static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq,
struct ibv_srq_attr *srq_attr,
int srq_attr_mask,
struct ibv_create_qp *cmd, size_t cmd_size,
struct ibv_create_qp_resp *resp, size_t resp_size)
{
+ struct ibv_context *context;
+
IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
cmd->user_handle = (uintptr_t) qp;
- cmd->pd_handle = pd->handle;
- cmd->send_cq_handle = attr->send_cq->handle;
- cmd->recv_cq_handle = attr->recv_cq->handle;
- cmd->srq_handle = attr->srq ? attr->srq->handle : 0;
+
+ if (attr->qp_type == IBV_QPT_XRC_RECV) {
+ context = attr->ext.xrc_recv.xrcd->context;
+ cmd->pd_handle = attr->ext.xrc_recv.xrcd->handle;
+ } else {
+ context = pd->context;
+ cmd->pd_handle = pd->handle;
+ cmd->send_cq_handle = attr->send_cq->handle;
+
+ if (attr->qp_type != IBV_QPT_XRC_SEND) {
+ cmd->recv_cq_handle = attr->recv_cq->handle;
+ cmd->srq_handle = attr->srq ? attr->srq->handle : 0;
+ }
+ }
+
cmd->max_send_wr = attr->cap.max_send_wr;
cmd->max_recv_wr = attr->cap.max_recv_wr;
cmd->max_send_sge = attr->cap.max_send_sge;
cmd->is_srq = !!attr->srq;
cmd->reserved = 0;
- if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
return errno;
(void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
qp->handle = resp->qp_handle;
qp->qp_num = resp->qpn;
- qp->context = pd->context;
+ qp->context = context;
if (abi_ver > 3) {
attr->cap.max_recv_sge = resp->max_recv_sge;
ibv_port_state_str;
ibv_event_type_str;
ibv_wc_status_str;
+
+ ibv_cmd_open_xrcd;
+ ibv_cmd_close_xrcd;
+ ibv_cmd_create_xsrq;
+ ibv_open_xrcd;
+ ibv_close_xrcd;
+ ibv_create_xsrq;
+
} IBVERBS_1.0;
}
default_symver(__ibv_dealloc_pd, ibv_dealloc_pd);
+struct ibv_xrcd *__ibv_open_xrcd(struct ibv_context *context, int fd, int oflags)
+{
+ struct ibv_xrc_ops *ops;
+ struct ibv_xrcd *xrcd;
+
+ ops = ibv_get_ext_ops(context, IBV_XRC_OPS);
+ if (!ops || !ops->open_xrcd)
+ return NULL;
+
+ xrcd = ops->open_xrcd(context, fd, oflags);
+ if (xrcd)
+ xrcd->context = context;
+
+ return xrcd;
+}
+default_symver(__ibv_open_xrcd, ibv_open_xrcd);
+
+int __ibv_close_xrcd(struct ibv_xrcd *xrcd)
+{
+ struct ibv_xrc_ops *ops;
+
+ ops = ibv_get_ext_ops(xrcd->context, IBV_XRC_OPS);
+ return ops->close_xrcd(xrcd);
+}
+default_symver(__ibv_close_xrcd, ibv_close_xrcd);
+
struct ibv_mr *__ibv_reg_mr(struct ibv_pd *pd, void *addr,
size_t length, int access)
{
}
default_symver(__ibv_ack_cq_events, ibv_ack_cq_events);
+/*
+ * Existing apps may be using an older, smaller version of srq_init_attr.
+ */
struct ibv_srq *__ibv_create_srq(struct ibv_pd *pd,
struct ibv_srq_init_attr *srq_init_attr)
{
+ struct ibv_srq_init_attr attr;
struct ibv_srq *srq;
- if (!pd->context->ops.create_srq)
+ attr.srq_context = srq_init_attr->srq_context;
+ attr.attr = srq_init_attr->attr;
+ attr.srq_type = IBV_SRQT_BASIC;
+
+ srq = ibv_create_xsrq(pd, &attr);
+ if (srq)
+ srq_init_attr->attr = attr.attr;
+
+ return srq;
+}
+default_symver(__ibv_create_srq, ibv_create_srq);
+
+struct ibv_srq *__ibv_create_srq_ex(struct ibv_pd *pd,
+ struct ibv_srq_init_attr_ex *srq_init_attr_ex)
+{
+ struct verbs_context *vcontext;
+ struct ibv_srq *srq;
+
+ vcontext = verbs_get_ctx(pd->context);
+ if (!vcontext || !vcontext->create_srq_ex)
+ return NULL;
+
+ if (!pd->context->ops.create_srq ||
+ (srq_init_attr->srq_type != IBV_SRQT_BASIC &&
+ !ibv_get_ext_support(pd->context->device)))
return NULL;
srq = pd->context->ops.create_srq(pd, srq_init_attr);
srq->events_completed = 0;
pthread_mutex_init(&srq->mutex, NULL);
pthread_cond_init(&srq->cond, NULL);
+
+ if (srq_init_attr->srq_type == IBV_SRQT_XRC) {
+ srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
+ srq->ext.xrc.cq = srq_init_attr->ext.xrc.cq;
+ }
+
+ if (ibv_get_ext_support(pd->context->device))
+ srq->srq_type = srq_init_attr->srq_type;
}
return srq;
}
-default_symver(__ibv_create_srq, ibv_create_srq);
+default_symver(__ibv_create_xsrq, ibv_create_xsrq);
int __ibv_modify_srq(struct ibv_srq *srq,
struct ibv_srq_attr *srq_attr,
struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr)
{
- struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
+ struct ibv_context *context;
+ struct ibv_qp *qp;
+ context = pd ? pd->context : qp_init_attr->ext.xrc_recv.xrcd->context;
+ qp = context->ops.create_qp(pd, qp_init_attr);
if (qp) {
- qp->context = pd->context;
- qp->qp_context = qp_init_attr->qp_context;
- qp->pd = pd;
- qp->send_cq = qp_init_attr->send_cq;
- qp->recv_cq = qp_init_attr->recv_cq;
- qp->srq = qp_init_attr->srq;
+ qp->context = context;
+ qp->qp_context = qp_init_attr->qp_context;
+
+ if (qp_init_attr->qp_type == IBV_QPT_XRC_RECV) {
+ qp->pd = NULL;
+ qp->send_cq = qp->recv_cq = NULL;
+ qp->srq = NULL;
+ qp->ext.xrc_recv.xrcd = qp_init_attr->ext.xrc_recv.xrcd;
+ } else {
+ if (qp_init_attr->qp_type == IBV_QPT_XRC_SEND) {
+ qp->recv_cq = NULL;
+ qp->srq = NULL;
+ } else {
+ qp->recv_cq = qp_init_attr->recv_cq;
+ qp->srq = qp_init_attr->srq;
+ }
+
+ qp->pd = pd;
+ qp->send_cq = qp_init_attr->send_cq;
+ }
+
qp->qp_type = qp_init_attr->qp_type;
qp->state = IBV_QPS_RESET;
qp->events_completed = 0;
projects / ~shefty / libibverbs.git / commitdiff