+2005-09-25 Roland Dreier <roland@cisco.com>
+
+ * examples/rc_pingpong.c, examples/srq_pingpong.c,
+ examples/uc_pingpong.c, examples/ud_pingpong.c: Update to match
+ new completion channel and CQ creation API.
+
+ * include/infiniband/driver.h, include/infiniband/verbs.h,
+ src/device.c, src/ibverbs.h, src/verbs.c, src/cmd.c: Add notion of
+ "completion channel" that allows consumers to dynamically create
+ and destroy file descriptors for retrieving completion events.
+ Completion channels are handled natively with kernel ABI version 3
+ and simulated with backwards compatibility implementations for ABI
+ versions 1 and 2.
+
+ * include/infiniband/kern-abi.h: Update to match kernel ABI
+ version 3.
+
2005-09-07 Roland Dreier <roland@cisco.com>
* src/device.c (ibv_get_device_guid): Use htonll() instead of
man/ibv_srq_pingpong.1
DEBIAN = debian/changelog debian/compat debian/control debian/copyright \
- debian/ibverbs-examples.install debian/libibverbs1.install \
+ debian/ibverbs-utils.install debian/libibverbs1.install \
debian/libibverbs-dev.install debian/rules
EXTRA_DIST = include/infiniband/driver.h include/infiniband/kern-abi.h \
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ(2.57)
-AC_INIT(libibverbs, 1.0-rc2, openib-general@openib.org)
+AC_INIT(libibverbs, 1.0-rc3, openib-general@openib.org)
AC_CONFIG_SRCDIR([src/ibverbs.h])
AC_CONFIG_AUX_DIR(config)
AM_CONFIG_HEADER(config.h)
-AM_INIT_AUTOMAKE(libibverbs, 1.0-rc2)
+AM_INIT_AUTOMAKE(libibverbs, 1.0-rc3)
AM_PROG_LIBTOOL
-libibverbs (1.0-rc2-1) unstable; urgency=low
+libibverbs (1.0-rc3) unstable; urgency=low
* Initial Release. (Closes: #325752)
It contains the header files and static libraries (optionally)
needed for compiling.
-Package: libibverbs-dbg
+Package: libibverbs1-dbg
Section: libdevel
Priority: extra
Architecture: any
Source:
It was downloaded from the OpenIB subversion repository at
-<https://openib.org/svn/gen2/trunk/src/userspace/libibverbs>
+<https://openib.org/downloads.html>
Authors:
The complete list of upstream authors is in the file
+++ /dev/null
-usr/bin
-usr/share/man/man1
--- /dev/null
+usr/bin
+usr/share/man/man1
static int page_size;
struct pingpong_context {
- struct ibv_context *context;
- struct ibv_pd *pd;
- struct ibv_mr *mr;
- struct ibv_cq *cq;
- struct ibv_qp *qp;
- void *buf;
- int size;
- int rx_depth;
+ struct ibv_context *context;
+ struct ibv_comp_channel *channel;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ void *buf;
+ int size;
+ int rx_depth;
};
struct pingpong_dest {
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
- int rx_depth, int port)
+ int rx_depth, int port,
+ int use_event)
{
struct pingpong_context *ctx;
return NULL;
}
+ if (use_event) {
+ ctx->channel = ibv_create_comp_channel(ctx->context);
+ if (!ctx->channel) {
+ fprintf(stderr, "Couldn't create completion channel\n");
+ return NULL;
+ }
+ } else
+ ctx->channel = NULL;
+
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return NULL;
}
- ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL);
+ ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
+ ctx->channel, 0);
if (!ctx->cq) {
fprintf(stderr, "Couldn't create CQ\n");
return NULL;
}
}
- ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port);
+ ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
if (!ctx)
return 1;
struct ibv_cq *ev_cq;
void *ev_ctx;
- if (ibv_get_cq_event(ctx->context, 0, &ev_cq, &ev_ctx)) {
+ if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
fprintf(stderr, "Failed to get cq_event\n");
return 1;
}
static int page_size;
struct pingpong_context {
- struct ibv_context *context;
- struct ibv_pd *pd;
- struct ibv_mr *mr;
- struct ibv_cq *cq;
- struct ibv_srq *srq;
- struct ibv_qp *qp[MAX_QP];
- void *buf;
- int size;
- int num_qp;
- int rx_depth;
+ struct ibv_context *context;
+ struct ibv_comp_channel *channel;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *cq;
+ struct ibv_srq *srq;
+ struct ibv_qp *qp[MAX_QP];
+ void *buf;
+ int size;
+ int num_qp;
+ int rx_depth;
};
struct pingpong_dest {
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
- int num_qp, int rx_depth, int port)
+ int num_qp, int rx_depth, int port,
+ int use_event)
{
struct pingpong_context *ctx;
int i;
return NULL;
}
+ if (use_event) {
+ ctx->channel = ibv_create_comp_channel(ctx->context);
+ if (!ctx->channel) {
+ fprintf(stderr, "Couldn't create completion channel\n");
+ return NULL;
+ }
+ } else
+ ctx->channel = NULL;
+
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return NULL;
}
- ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL);
+ ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
+ ctx->channel, 0);
if (!ctx->cq) {
fprintf(stderr, "Couldn't create CQ\n");
return NULL;
}
}
- ctx = pp_init_ctx(ib_dev, size, num_qp, rx_depth, ib_port);
+ ctx = pp_init_ctx(ib_dev, size, num_qp, rx_depth, ib_port, use_event);
if (!ctx)
return 1;
struct ibv_cq *ev_cq;
void *ev_ctx;
- if (ibv_get_cq_event(ctx->context, 0, &ev_cq, &ev_ctx)) {
+ if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
fprintf(stderr, "Failed to get cq_event\n");
return 1;
}
static int page_size;
struct pingpong_context {
- struct ibv_context *context;
- struct ibv_pd *pd;
- struct ibv_mr *mr;
- struct ibv_cq *cq;
- struct ibv_qp *qp;
- void *buf;
- int size;
- int rx_depth;
+ struct ibv_context *context;
+ struct ibv_comp_channel *channel;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ void *buf;
+ int size;
+ int rx_depth;
};
struct pingpong_dest {
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
- int rx_depth, int port)
+ int rx_depth, int port,
+ int use_event)
{
struct pingpong_context *ctx;
return NULL;
}
+ if (use_event) {
+ ctx->channel = ibv_create_comp_channel(ctx->context);
+ if (!ctx->channel) {
+ fprintf(stderr, "Couldn't create completion channel\n");
+ return NULL;
+ }
+ } else
+ ctx->channel = NULL;
+
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return NULL;
}
- ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL);
+ ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
+ ctx->channel, 0);
if (!ctx->cq) {
fprintf(stderr, "Couldn't create CQ\n");
return NULL;
}
}
- ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port);
+ ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
if (!ctx)
return 1;
struct ibv_cq *ev_cq;
void *ev_ctx;
- if (ibv_get_cq_event(ctx->context, 0, &ev_cq, &ev_ctx)) {
+ if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
fprintf(stderr, "Failed to get cq_event\n");
return 1;
}
static int page_size;
struct pingpong_context {
- struct ibv_context *context;
- struct ibv_pd *pd;
- struct ibv_mr *mr;
- struct ibv_cq *cq;
- struct ibv_qp *qp;
- struct ibv_ah *ah;
- void *buf;
- int size;
- int rx_depth;
+ struct ibv_context *context;
+ struct ibv_comp_channel *channel;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ struct ibv_ah *ah;
+ void *buf;
+ int size;
+ int rx_depth;
};
struct pingpong_dest {
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
- int rx_depth, int port)
+ int rx_depth, int port,
+ int use_event)
{
struct pingpong_context *ctx;
return NULL;
}
+ if (use_event) {
+ ctx->channel = ibv_create_comp_channel(ctx->context);
+ if (!ctx->channel) {
+ fprintf(stderr, "Couldn't create completion channel\n");
+ return NULL;
+ }
+ } else
+ ctx->channel = NULL;
+
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return NULL;
}
- ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL);
+ ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
+ ctx->channel, 0);
if (!ctx->cq) {
fprintf(stderr, "Couldn't create CQ\n");
return NULL;
}
}
- ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port);
+ ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
if (!ctx)
return 1;
struct ibv_cq *ev_cq;
void *ev_ctx;
- if (ibv_get_cq_event(ctx->context, 0, &ev_cq, &ev_ctx)) {
+ if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
fprintf(stderr, "Failed to get cq_event\n");
return 1;
}
typedef struct ibv_device *(*ibv_driver_init_func)(struct sysfs_class_device *);
-extern int ibv_cmd_get_context(int num_comp, struct ibv_context *context,
- struct ibv_get_context *cmd, size_t cmd_size,
- struct ibv_get_context_resp *resp, size_t resp_size);
+extern int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
+ size_t cmd_size, struct ibv_get_context_resp *resp,
+ size_t resp_size);
extern int ibv_cmd_query_device(struct ibv_context *context,
struct ibv_device_attr *device_attr,
struct ibv_query_device *cmd, size_t cmd_size);
size_t cmd_size);
extern int ibv_cmd_dereg_mr(struct ibv_mr *mr);
extern int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
- struct ibv_cq *cq,
+ struct ibv_comp_channel *channel,
+ int comp_vector, struct ibv_cq *cq,
struct ibv_create_cq *cmd, size_t cmd_size,
struct ibv_create_cq_resp *resp, size_t resp_size);
extern int ibv_cmd_destroy_cq(struct ibv_cq *cq);
* The minimum and maximum kernel ABI that we can handle.
*/
#define IB_USER_VERBS_MIN_ABI_VERSION 1
-#define IB_USER_VERBS_MAX_ABI_VERSION 2
+#define IB_USER_VERBS_MAX_ABI_VERSION 3
enum {
- IB_USER_VERBS_CMD_QUERY_PARAMS,
IB_USER_VERBS_CMD_GET_CONTEXT,
IB_USER_VERBS_CMD_QUERY_DEVICE,
IB_USER_VERBS_CMD_QUERY_PORT,
- IB_USER_VERBS_CMD_QUERY_GID,
- IB_USER_VERBS_CMD_QUERY_PKEY,
IB_USER_VERBS_CMD_ALLOC_PD,
IB_USER_VERBS_CMD_DEALLOC_PD,
IB_USER_VERBS_CMD_CREATE_AH,
IB_USER_VERBS_CMD_ALLOC_MW,
IB_USER_VERBS_CMD_BIND_MW,
IB_USER_VERBS_CMD_DEALLOC_MW,
+ IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
IB_USER_VERBS_CMD_CREATE_CQ,
IB_USER_VERBS_CMD_RESIZE_CQ,
IB_USER_VERBS_CMD_DESTROY_CQ,
__u16 in_words;
__u16 out_words;
__u64 response;
- __u64 cq_fd_tab;
__u64 driver_data[0];
};
struct ibv_get_context_resp {
__u32 async_fd;
- __u32 reserved;
+ __u32 num_comp_vectors;
};
struct ibv_query_device {
__u8 reserved[3];
};
-struct ibv_query_gid {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u64 response;
- __u8 port_num;
- __u8 index;
- __u8 reserved[6];
- __u64 driver_data[0];
-};
-
-struct ibv_query_gid_resp {
- __u8 gid[16];
-};
-
-struct ibv_query_pkey {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u64 response;
- __u8 port_num;
- __u8 index;
- __u8 reserved[6];
- __u64 driver_data[0];
-};
-
-struct ibv_query_pkey_resp {
- __u16 pkey;
- __u16 reserved;
-};
-
struct ibv_alloc_pd {
__u32 command;
__u16 in_words;
__u32 mr_handle;
};
+struct ibv_create_comp_channel {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+};
+
+struct ibv_create_comp_channel_resp {
+ __u32 fd;
+};
+
struct ibv_create_cq {
__u32 command;
__u16 in_words;
__u64 response;
__u64 user_handle;
__u32 cqe;
- __u32 event_handler;
+ __u32 comp_vector;
+ __s32 comp_channel;
+ __u32 reserved;
__u64 driver_data[0];
};
__u32 cqe;
};
-struct ibv_destroy_cq_v1 {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u32 cq_handle;
-};
-
struct ibv_destroy_cq {
__u32 command;
__u16 in_words;
__u64 driver_data[0];
};
-struct ibv_destroy_qp_v1 {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u32 qp_handle;
-};
-
struct ibv_destroy_qp {
__u32 command;
__u16 in_words;
__u64 driver_data[0];
};
+struct ibv_destroy_srq {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u32 srq_handle;
+ __u32 reserved;
+};
+
+struct ibv_destroy_srq_resp {
+ __u32 events_reported;
+};
+
+/*
+ * Compatibility with older ABI versions
+ */
+
+enum {
+ IB_USER_VERBS_CMD_QUERY_PARAMS_V2,
+ IB_USER_VERBS_CMD_GET_CONTEXT_V2,
+ IB_USER_VERBS_CMD_QUERY_DEVICE_V2,
+ IB_USER_VERBS_CMD_QUERY_PORT_V2,
+ IB_USER_VERBS_CMD_QUERY_GID_V2,
+ IB_USER_VERBS_CMD_QUERY_PKEY_V2,
+ IB_USER_VERBS_CMD_ALLOC_PD_V2,
+ IB_USER_VERBS_CMD_DEALLOC_PD_V2,
+ IB_USER_VERBS_CMD_CREATE_AH_V2,
+ IB_USER_VERBS_CMD_MODIFY_AH_V2,
+ IB_USER_VERBS_CMD_QUERY_AH_V2,
+ IB_USER_VERBS_CMD_DESTROY_AH_V2,
+ IB_USER_VERBS_CMD_REG_MR_V2,
+ IB_USER_VERBS_CMD_REG_SMR_V2,
+ IB_USER_VERBS_CMD_REREG_MR_V2,
+ IB_USER_VERBS_CMD_QUERY_MR_V2,
+ IB_USER_VERBS_CMD_DEREG_MR_V2,
+ IB_USER_VERBS_CMD_ALLOC_MW_V2,
+ IB_USER_VERBS_CMD_BIND_MW_V2,
+ IB_USER_VERBS_CMD_DEALLOC_MW_V2,
+ IB_USER_VERBS_CMD_CREATE_CQ_V2,
+ IB_USER_VERBS_CMD_RESIZE_CQ_V2,
+ IB_USER_VERBS_CMD_DESTROY_CQ_V2,
+ IB_USER_VERBS_CMD_POLL_CQ_V2,
+ IB_USER_VERBS_CMD_PEEK_CQ_V2,
+ IB_USER_VERBS_CMD_REQ_NOTIFY_CQ_V2,
+ IB_USER_VERBS_CMD_CREATE_QP_V2,
+ IB_USER_VERBS_CMD_QUERY_QP_V2,
+ IB_USER_VERBS_CMD_MODIFY_QP_V2,
+ IB_USER_VERBS_CMD_DESTROY_QP_V2,
+ IB_USER_VERBS_CMD_POST_SEND_V2,
+ IB_USER_VERBS_CMD_POST_RECV_V2,
+ IB_USER_VERBS_CMD_ATTACH_MCAST_V2,
+ IB_USER_VERBS_CMD_DETACH_MCAST_V2,
+ IB_USER_VERBS_CMD_CREATE_SRQ_V2,
+ IB_USER_VERBS_CMD_MODIFY_SRQ_V2,
+ IB_USER_VERBS_CMD_QUERY_SRQ_V2,
+ IB_USER_VERBS_CMD_DESTROY_SRQ_V2,
+ IB_USER_VERBS_CMD_POST_SRQ_RECV_V2,
+ /*
+ * Set commands that didn't exist to -1 so our compile-time
+ * trick opcodes in IBV_INIT_CMD() doesn't break.
+ */
+ IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
+};
+
+struct ibv_destroy_cq_v1 {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u32 cq_handle;
+};
+
+struct ibv_destroy_qp_v1 {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u32 qp_handle;
+};
+
struct ibv_destroy_srq_v1 {
__u32 command;
__u16 in_words;
__u32 srq_handle;
};
-struct ibv_destroy_srq {
+struct ibv_get_context_v2 {
__u32 command;
__u16 in_words;
__u16 out_words;
__u64 response;
- __u32 srq_handle;
- __u32 reserved;
+ __u64 cq_fd_tab;
+ __u64 driver_data[0];
};
-struct ibv_destroy_srq_resp {
- __u32 events_reported;
+struct ibv_create_cq_v2 {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u64 user_handle;
+ __u32 cqe;
+ __u32 event_handler;
+ __u64 driver_data[0];
};
#endif /* KERN_ABI_H */
uint32_t events_completed;
};
+struct ibv_comp_channel {
+ int fd;
+};
+
struct ibv_cq {
struct ibv_context *context;
void *cq_context;
struct ibv_context;
struct ibv_device_ops {
- struct ibv_context * (*alloc_context)(struct ibv_device *device,
- int num_comp, int cmd_fd);
+ struct ibv_context * (*alloc_context)(struct ibv_device *device, int cmd_fd);
void (*free_context)(struct ibv_context *context);
};
struct ibv_device_attr *device_attr);
int (*query_port)(struct ibv_context *context, uint8_t port_num,
struct ibv_port_attr *port_attr);
- int (*query_gid)(struct ibv_context *context, uint8_t port_num,
- int index, union ibv_gid *gid);
- int (*query_pkey)(struct ibv_context *context, uint8_t port_num,
- int index, uint16_t *pkey);
struct ibv_pd * (*alloc_pd)(struct ibv_context *context);
int (*dealloc_pd)(struct ibv_pd *pd);
struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length,
enum ibv_access_flags access);
int (*dereg_mr)(struct ibv_mr *mr);
- struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe);
+ struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel,
+ int comp_vector);
int (*poll_cq)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc);
- int (*req_notify_cq)(struct ibv_cq *cq, int solicited);
+ int (*req_notify_cq)(struct ibv_cq *cq, int solicited_only);
void (*cq_event)(struct ibv_cq *cq);
int (*destroy_cq)(struct ibv_cq *cq);
struct ibv_srq * (*create_srq)(struct ibv_pd *pd,
struct ibv_context_ops ops;
int cmd_fd;
int async_fd;
- int num_comp;
- int cq_fd[1];
+ int num_comp_vectors;
+ void *abi_compat;
};
/**
*/
extern int ibv_dereg_mr(struct ibv_mr *mr);
+/**
+ * ibv_create_comp_channel - Create a completion event channel
+ */
+extern struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context);
+
+/**
+ * ibv_destroy_comp_channel - Destroy a completion event channel
+ */
+extern int ibv_destroy_comp_channel(struct ibv_comp_channel *channel);
+
/**
* ibv_create_cq - Create a completion queue
+ * @context - Context CQ will be attached to
+ * @cqe - Minimum number of entries required for CQ
+ * @cq_context - Consumer-supplied context returned for completion events
+ * @channel - Completion channel where completion events will be queued.
+ * May be NULL if completion events will not be used.
+ * @comp_vector - Completion vector used to signal completion events.
+ * Must be >= 0 and < context->num_comp_vectors.
*/
extern struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
- void *cq_context);
+ void *cq_context,
+ struct ibv_comp_channel *channel,
+ int comp_vector);
/**
* ibv_destroy_cq - Destroy a completion queue
/**
* ibv_get_cq_event - Read next CQ event
- * @context: Context to get CQ event for
- * @comp_num: Index of completion event to check. Must be >= 0 and
- * <= context->num_comp.
+ * @channel: Channel to get next event from.
* @cq: Used to return pointer to CQ.
* @cq_context: Used to return consumer-supplied CQ context.
*
* All completion events returned by ibv_get_cq_event() must
* eventually be acknowledged with ibv_ack_cq_events().
*/
-extern int ibv_get_cq_event(struct ibv_context *context, int comp_num,
+extern int ibv_get_cq_event(struct ibv_comp_channel *channel,
struct ibv_cq **cq, void **cq_context);
/**
/**
* ibv_req_notify_cq - Request completion notification on a CQ.
*/
-static inline int ibv_req_notify_cq(struct ibv_cq *cq, int solicited)
+static inline int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only)
{
- return cq->context->ops.req_notify_cq(cq, solicited);
+ return cq->context->ops.req_notify_cq(cq, solicited_only);
}
/**
#include "ibverbs.h"
-int ibv_cmd_get_context(int num_comp, struct ibv_context *context,
- struct ibv_get_context *cmd, size_t cmd_size,
- struct ibv_get_context_resp *resp, size_t resp_size)
+static int ibv_cmd_get_context_v2(struct ibv_context *context,
+ struct ibv_get_context *new_cmd,
+ size_t new_cmd_size,
+ struct ibv_get_context_resp *resp,
+ size_t resp_size)
{
- uint32_t *cq_fd_tab;
- int i;
+ struct ibv_abi_compat_v2 *t;
+ struct ibv_get_context_v2 *cmd;
+ size_t cmd_size;
+ uint32_t cq_fd;
+
+ t = malloc(sizeof *t);
+ if (!t)
+ return ENOMEM;
+ pthread_mutex_init(&t->in_use, NULL);
+
+ cmd_size = sizeof cmd + new_cmd_size - sizeof *new_cmd;
+ cmd = alloca(cmd_size);
+ memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
- cq_fd_tab = alloca(num_comp * sizeof (uint32_t));
IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
+ cmd->cq_fd_tab = (uintptr_t) &cq_fd;
- cmd->cq_fd_tab = (uintptr_t) cq_fd_tab;
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ context->async_fd = resp->async_fd;
+ context->num_comp_vectors = 1;
+ t->channel.fd = cq_fd;
+ context->abi_compat = t;
+
+ return 0;
+}
+
+int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
+ size_t cmd_size, struct ibv_get_context_resp *resp,
+ size_t resp_size)
+{
+ if (abi_ver <= 2)
+ return ibv_cmd_get_context_v2(context, cmd, cmd_size, resp, resp_size);
+
+ IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
return errno;
- context->async_fd = resp->async_fd;
- for (i = 0; i < num_comp; ++i)
- context->cq_fd[i] = cq_fd_tab[i];
+ context->async_fd = resp->async_fd;
+ context->num_comp_vectors = resp->num_comp_vectors;
return 0;
}
return 0;
}
-int ibv_cmd_query_gid(struct ibv_context *context, uint8_t port_num,
- int index, union ibv_gid *gid)
-{
- struct ibv_query_gid cmd;
- struct ibv_query_gid_resp resp;
-
- IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_GID, &resp, sizeof resp);
- cmd.port_num = port_num;
- cmd.index = index;
-
- if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
- return errno;
-
- memcpy(gid->raw, resp.gid, 16);
-
- return 0;
-}
-
-int ibv_cmd_query_pkey(struct ibv_context *context, uint8_t port_num,
- int index, uint16_t *pkey)
-{
- struct ibv_query_pkey cmd;
- struct ibv_query_pkey_resp resp;
-
- IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_PKEY, &resp, sizeof resp);
- cmd.port_num = port_num;
- cmd.index = index;
-
- if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
- return errno;
-
- *pkey = resp.pkey;
-
- return 0;
-}
-
int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
struct ibv_alloc_pd *cmd, size_t cmd_size,
struct ibv_alloc_pd_resp *resp, size_t resp_size)
return 0;
}
+static int ibv_cmd_create_cq_v2(struct ibv_context *context, int cqe,
+ struct ibv_cq *cq,
+ struct ibv_create_cq *new_cmd, size_t new_cmd_size,
+ struct ibv_create_cq_resp *resp, size_t resp_size)
+{
+ struct ibv_create_cq_v2 *cmd;
+ size_t cmd_size;
+
+ cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
+ cmd = alloca(cmd_size);
+ memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
+
+ IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size);
+ cmd->user_handle = (uintptr_t) cq;
+ cmd->cqe = cqe;
+ cmd->event_handler = 0;
+
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ cq->handle = resp->cq_handle;
+ cq->cqe = resp->cqe;
+
+ return 0;
+}
+
int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
- struct ibv_cq *cq,
+ struct ibv_comp_channel *channel,
+ int comp_vector, struct ibv_cq *cq,
struct ibv_create_cq *cmd, size_t cmd_size,
struct ibv_create_cq_resp *resp, size_t resp_size)
{
+ if (abi_ver <= 2)
+ return ibv_cmd_create_cq_v2(context, cqe, cq,
+ cmd, cmd_size, resp, resp_size);
+
IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size);
cmd->user_handle = (uintptr_t) cq;
cmd->cqe = cqe;
- cmd->event_handler = 0;
+ cmd->comp_vector = comp_vector;
+ cmd->comp_channel = channel ? channel->fd : -1;
+ cmd->reserved = 0;
if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
return errno;
char *devpath;
int cmd_fd;
struct ibv_context *context;
- struct ibv_query_params cmd;
- struct ibv_query_params_resp resp;
asprintf(&devpath, "/dev/infiniband/%s", device->dev->name);
if (cmd_fd < 0)
return NULL;
- IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_PARAMS, &resp, sizeof resp);
- if (write(cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
- goto err;
-
- context = device->ops.alloc_context(device, resp.num_cq_events, cmd_fd);
+ context = device->ops.alloc_context(device, cmd_fd);
if (!context)
goto err;
- context->device = device;
- context->cmd_fd = cmd_fd;
- context->num_comp = resp.num_cq_events;
+ context->device = device;
+ context->cmd_fd = cmd_fd;
return context;
int ibv_close_device(struct ibv_context *context)
{
- int i;
+ int async_fd = context->async_fd;
+ int cmd_fd = context->cmd_fd;
+ int cq_fd = -1;
+
+ if (abi_ver <= 2) {
+ struct ibv_abi_compat_v2 *t = context->abi_compat;
+ cq_fd = t->channel.fd;
+ free(context->abi_compat);
+ }
context->device->ops.free_context(context);
- close(context->async_fd);
- for (i = 0; i < context->num_comp; ++i)
- close(context->cq_fd[i]);
- close(context->cmd_fd);
+ close(async_fd);
+ close(cmd_fd);
+ if (abi_ver <= 2)
+ close(cq_fd);
return 0;
}
ibv_driver_init_func init_func;
};
+struct ibv_abi_compat_v2 {
+ struct ibv_comp_channel channel;
+ pthread_mutex_t in_use;
+};
+
extern HIDDEN int abi_ver;
extern struct dlist *ibverbs_init(void);
extern int ibv_lock_range(void *base, size_t size);
extern int ibv_unlock_range(void *base, size_t size);
-#define IBV_INIT_CMD(cmd, size, opcode) \
- do { \
- (cmd)->command = IB_USER_VERBS_CMD_##opcode; \
- (cmd)->in_words = (size) / 4; \
- (cmd)->out_words = 0; \
+#define IBV_INIT_CMD(cmd, size, opcode) \
+ do { \
+ if (abi_ver > 2) \
+ (cmd)->command = IB_USER_VERBS_CMD_##opcode; \
+ else \
+ (cmd)->command = IB_USER_VERBS_CMD_##opcode##_V2; \
+ (cmd)->in_words = (size) / 4; \
+ (cmd)->out_words = 0; \
} while (0)
-#define IBV_INIT_CMD_RESP(cmd, size, opcode, out, outsize) \
- do { \
- (cmd)->command = IB_USER_VERBS_CMD_##opcode; \
- (cmd)->in_words = (size) / 4; \
- (cmd)->out_words = (outsize) / 4; \
- (cmd)->response = (uintptr_t) (out); \
+#define IBV_INIT_CMD_RESP(cmd, size, opcode, out, outsize) \
+ do { \
+ if (abi_ver > 2) \
+ (cmd)->command = IB_USER_VERBS_CMD_##opcode; \
+ else \
+ (cmd)->command = IB_USER_VERBS_CMD_##opcode##_V2; \
+ (cmd)->in_words = (size) / 4; \
+ (cmd)->out_words = (outsize) / 4; \
+ (cmd)->response = (uintptr_t) (out); \
} while (0)
#endif /* IB_VERBS_H */
ibv_dealloc_pd;
ibv_reg_mr;
ibv_dereg_mr;
+ ibv_create_comp_channel;
+ ibv_destroy_comp_channel;
ibv_create_cq;
ibv_destroy_cq;
ibv_get_cq_event;
#endif /* HAVE_CONFIG_H */
#include <stdio.h>
+#include <netinet/in.h>
#include <unistd.h>
#include <errno.h>
int ibv_query_gid(struct ibv_context *context, uint8_t port_num,
int index, union ibv_gid *gid)
{
- return context->ops.query_gid(context, port_num, index, gid);
+ char *attr_name;
+ char attr[sizeof "0000:0000:0000:0000:0000:0000:0000:0000\0"];
+ uint16_t val;
+ int i;
+
+ asprintf(&attr_name, "%s/ports/%d/gids/%d",
+ context->device->ibdev->path, port_num, index);
+
+ if (sysfs_read_attribute_value(attr_name, attr, sizeof attr))
+ return -1;
+
+ for (i = 0; i < 8; ++i) {
+ if (sscanf(attr + i * 5, "%hx", &val) != 1)
+ return -1;
+ gid->raw[i * 2 ] = val >> 8;
+ gid->raw[i * 2 + 1] = val & 0xff;
+ }
+
+ return 0;
}
int ibv_query_pkey(struct ibv_context *context, uint8_t port_num,
int index, uint16_t *pkey)
{
- return context->ops.query_pkey(context, port_num, index, pkey);
+ char *attr_name;
+ char attr[sizeof "0x0000\0"];
+ uint16_t val;
+
+ asprintf(&attr_name, "%s/ports/%d/pkeys/%d",
+ context->device->ibdev->path, port_num, index);
+
+ if (sysfs_read_attribute_value(attr_name, attr, sizeof attr))
+ return -1;
+
+ if (sscanf(attr, "%hx", &val) != 1)
+ return -1;
+
+ *pkey = htons(val);
+ return 0;
}
struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
return mr->context->ops.dereg_mr(mr);
}
-struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
- void *cq_context)
+static struct ibv_comp_channel *ibv_create_comp_channel_v2(struct ibv_context *context)
+{
+ struct ibv_abi_compat_v2 *t = context->abi_compat;
+ static int warned;
+
+ if (!pthread_mutex_trylock(&t->in_use))
+ return &t->channel;
+
+ if (!warned) {
+ fprintf(stderr, PFX "Warning: kernel's ABI version %d limits capacity.\n"
+ " Only one completion channel can be created per context.\n",
+ abi_ver);
+ ++warned;
+ }
+
+ return NULL;
+}
+
+struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context)
{
- struct ibv_cq *cq = context->ops.create_cq(context, cqe);
+ struct ibv_comp_channel *channel;
+ struct ibv_create_comp_channel cmd;
+ struct ibv_create_comp_channel_resp resp;
+
+ if (abi_ver <= 2)
+ return ibv_create_comp_channel_v2(context);
+
+ channel = malloc(sizeof *channel);
+ if (!channel)
+ return NULL;
+
+ IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_COMP_CHANNEL, &resp, sizeof resp);
+ if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) {
+ free(channel);
+ return NULL;
+ }
+
+ channel->fd = resp.fd;
+
+ return channel;
+}
+
+static int ibv_destroy_comp_channel_v2(struct ibv_comp_channel *channel)
+{
+ struct ibv_abi_compat_v2 *t = (struct ibv_abi_compat_v2 *) channel;
+ pthread_mutex_unlock(&t->in_use);
+ return 0;
+}
+
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+ if (abi_ver <= 2)
+ return ibv_destroy_comp_channel_v2(channel);
+
+ close(channel->fd);
+ free(channel);
+
+ return 0;
+}
+
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, void *cq_context,
+ struct ibv_comp_channel *channel, int comp_vector)
+{
+ struct ibv_cq *cq = context->ops.create_cq(context, cqe, channel,
+ comp_vector);
if (cq) {
cq->context = context;
}
-int ibv_get_cq_event(struct ibv_context *context, int comp_num,
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
struct ibv_cq **cq, void **cq_context)
{
struct ibv_comp_event ev;
- if (comp_num < 0 || comp_num >= context->num_comp)
- return -1;
-
- if (read(context->cq_fd[comp_num], &ev, sizeof ev) != sizeof ev)
+ if (read(channel->fd, &ev, sizeof ev) != sizeof ev)
return -1;
*cq = (struct ibv_cq *) (uintptr_t) ev.cq_handle;
projects / ~shefty / libibverbs.git / commitdiff