+Sean Hefty <sean.hefty@intel.com>
Libor Michalek <libor@topspin.com>
src_libibcm_la_SOURCES = src/cm.c
src_libibcm_la_LDFLAGS = -avoid-version -module $(ucm_version_script)
-bin_PROGRAMS = examples/ucm_simple
-examples_ucm_simple_SOURCES = examples/simple.c
-examples_ucm_simple_LDADD = $(top_builddir)/src/libibcm.la
+bin_PROGRAMS = examples/ucmpost
+examples_ucmpost_SOURCES = examples/cmpost.c
+examples_ucmpost_LDADD = $(top_builddir)/src/libibcm.la \
+ $(libdir)/libibverbs.la \
+ $(libdir)/libibat.la
libibcmincludedir = $(includedir)/infiniband
--- /dev/null
+/*
+ * Copyright (c) 2004, 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <byteswap.h>
+
+#include <infiniband/cm.h>
+#include <infiniband/at.h>
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
+static inline uint32_t cpu_to_be32(uint32_t x) { return x; }
+#else
+static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
+static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
+#endif
+
+/*
+ * To execute:
+ * Server: ucmpost
+ * Client: ucmpost server
+ */
+
+struct cmtest {
+ struct ibv_device *device;
+ struct ibv_context *verbs;
+ struct ibv_pd *pd;
+
+ /* cm info */
+ struct ib_sa_path_rec path_rec;
+
+ struct cmtest_node *nodes;
+ int conn_index;
+ int connects_left;
+ int disconnects_left;
+
+ /* memory region info */
+ struct ibv_mr *mr;
+ void *mem;
+};
+
+static struct cmtest test;
+static int message_count = 10;
+static int message_size = 100;
+static int connections = 1;
+static int is_server = 1;
+
+struct cmtest_node {
+ int id;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ struct ib_cm_id *cm_id;
+ int connected;
+};
+
+static int post_recvs(struct cmtest_node *node)
+{
+ struct ibv_recv_wr recv_wr, *recv_failure;
+ struct ibv_sge sge;
+ int i, ret = 0;
+
+ if (!message_count)
+ return 0;
+
+ recv_wr.next = NULL;
+ recv_wr.sg_list = &sge;
+ recv_wr.num_sge = 1;
+ recv_wr.wr_id = (uintptr_t) node;
+
+ sge.length = message_size;
+ sge.lkey = test.mr->lkey;
+ sge.addr = (uintptr_t) test.mem;
+
+ for (i = 0; i < message_count && !ret; i++ ) {
+ ret = ibv_post_recv(node->qp, &recv_wr, &recv_failure);
+ if (ret) {
+ printf("failed to post receives: %d\n", ret);
+ break;
+ }
+ }
+ return ret;
+}
+
+static int modify_to_rtr(struct cmtest_node *node)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ qp_attr.qp_state = IBV_QPS_INIT;
+ ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+ if (ret) {
+ printf("failed to init QP attr for INIT: %d\n", ret);
+ return ret;
+ }
+ ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+ if (ret) {
+ printf("failed to modify QP to INIT: %d\n", ret);
+ return ret;
+ }
+ qp_attr.qp_state = IBV_QPS_RTR;
+ ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+ if (ret) {
+ printf("failed to init QP attr for RTR: %d\n", ret);
+ return ret;
+ }
+ qp_attr.rq_psn = node->qp->qp_num;
+ ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+ if (ret) {
+ printf("failed to modify QP to RTR: %d\n", ret);
+ return ret;
+ }
+ return 0;
+}
+
+static int modify_to_rts(struct cmtest_node *node)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ qp_attr.qp_state = IBV_QPS_RTS;
+ ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+ if (ret) {
+ printf("failed to init QP attr for RTS: %d\n", ret);
+ return ret;
+ }
+ ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+ if (ret) {
+ printf("failed to modify QP to RTS: %d\n", ret);
+ return ret;
+ }
+ return 0;
+}
+
+static void req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+ struct cmtest_node *node;
+ struct ib_cm_req_event_param *req;
+ struct ib_cm_rep_param rep;
+ int ret;
+
+ if (test.conn_index == connections)
+ goto error1;
+ node = &test.nodes[test.conn_index++];
+
+ node->cm_id = cm_id;
+ cm_id->context = node;
+
+ ret = modify_to_rtr(node);
+ if (ret)
+ goto error2;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto error2;
+
+ req = &event->param.req_rcvd;
+ memset(&rep, 0, sizeof rep);
+ rep.qp_num = node->qp->qp_num;
+ rep.srq = (node->qp->srq != NULL);
+ rep.starting_psn = node->qp->qp_num;
+ rep.responder_resources = req->responder_resources;
+ rep.initiator_depth = req->initiator_depth;
+ rep.target_ack_delay = 20;
+ rep.flow_control = req->flow_control;
+ rep.rnr_retry_count = req->rnr_retry_count;
+
+ ret = ib_cm_send_rep(cm_id, &rep);
+ if (ret) {
+ printf("failed to send CM REP: %d\n", ret);
+ goto error2;
+ }
+ return;
+error2:
+ test.disconnects_left--;
+ test.connects_left--;
+error1:
+ printf("failing connection request\n");
+ ib_cm_send_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
+}
+
+static void rep_handler(struct cmtest_node *node, struct ib_cm_event *event)
+{
+ int ret;
+
+ ret = modify_to_rtr(node);
+ if (ret)
+ goto error;
+
+ ret = modify_to_rts(node);
+ if (ret)
+ goto error;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto error;
+
+ ret = ib_cm_send_rtu(node->cm_id, NULL, 0);
+ if (ret) {
+ printf("failed to send CM RTU: %d\n", ret);
+ goto error;
+ }
+ node->connected = 1;
+ test.connects_left--;
+ return;
+error:
+ printf("failing connection reply\n");
+ ib_cm_send_rej(node->cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
+ test.disconnects_left--;
+ test.connects_left--;
+}
+
+static void rtu_handler(struct cmtest_node *node)
+{
+ int ret;
+
+ ret = modify_to_rts(node);
+ if (ret)
+ goto error;
+
+ node->connected = 1;
+ test.connects_left--;
+ return;
+error:
+ printf("aborting connection - disconnecting\n");
+ ib_cm_send_dreq(node->cm_id, NULL, 0);
+ test.disconnects_left--;
+ test.connects_left--;
+}
+
+static void cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+ struct cmtest_node *node = cm_id->context;
+
+ switch (event->event) {
+ case IB_CM_REQ_RECEIVED:
+ req_handler(cm_id, event);
+ break;
+ case IB_CM_REP_RECEIVED:
+ rep_handler(node, event);
+ break;
+ case IB_CM_RTU_RECEIVED:
+ rtu_handler(node);
+ break;
+ case IB_CM_DREQ_RECEIVED:
+ node->connected = 0;
+ ib_cm_send_drep(node->cm_id, NULL, 0);
+ test.disconnects_left--;
+ break;
+ case IB_CM_DREP_RECEIVED:
+ test.disconnects_left--;
+ break;
+ case IB_CM_REJ_RECEIVED:
+ printf("Received REJ\n");
+ /* fall through */
+ case IB_CM_REQ_ERROR:
+ case IB_CM_REP_ERROR:
+ printf("Error sending REQ or REP\n");
+ test.disconnects_left--;
+ test.connects_left--;
+ break;
+ case IB_CM_DREQ_ERROR:
+ test.disconnects_left--;
+ printf("Error sending DREQ\n");
+ break;
+ default:
+ break;
+ }
+}
+
+static int init_node(struct cmtest_node *node, struct ibv_qp_init_attr *qp_attr)
+{
+ int cqe, ret;
+
+ if (!is_server) {
+ ret = ib_cm_create_id(&node->cm_id, node);
+ if (ret) {
+ printf("failed to create cm_id: %d\n", ret);
+ return ret;
+ }
+ }
+
+ cqe = message_count ? message_count * 2 : 2;
+ node->cq = ibv_create_cq(test.verbs, cqe, node);
+ if (!node->cq) {
+ printf("unable to create CQ\n");
+ goto error1;
+ }
+
+ qp_attr->send_cq = node->cq;
+ qp_attr->recv_cq = node->cq;
+ node->qp = ibv_create_qp(test.pd, qp_attr);
+ if (!node->qp) {
+ printf("unable to create QP\n");
+ goto error2;
+ }
+ return 0;
+error2:
+ ibv_destroy_cq(node->cq);
+error1:
+ if (!is_server)
+ ib_cm_destroy_id(node->cm_id);
+ return -1;
+}
+
+static void destroy_node(struct cmtest_node *node)
+{
+ ibv_destroy_qp(node->qp);
+ ibv_destroy_cq(node->cq);
+ if (node->cm_id)
+ ib_cm_destroy_id(node->cm_id);
+}
+
+static int create_nodes(void)
+{
+ struct ibv_qp_init_attr qp_attr;
+ int ret, i;
+
+ test.nodes = malloc(sizeof *test.nodes * connections);
+ if (!test.nodes) {
+ printf("unable to allocate memory for test nodes\n");
+ return -1;
+ }
+ memset(test.nodes, 0, sizeof *test.nodes * connections);
+
+ memset(&qp_attr, 0, sizeof qp_attr);
+ qp_attr.cap.max_send_wr = message_count ? message_count : 1;
+ qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
+ qp_attr.cap.max_send_sge = 1;
+ qp_attr.cap.max_recv_sge = 1;
+ qp_attr.qp_type = IBV_QPT_RC;
+
+ for (i = 0; i < connections; i++) {
+ test.nodes[i].id = i;
+ ret = init_node(&test.nodes[i], &qp_attr);
+ if (ret)
+ goto error;
+ }
+ return 0;
+error:
+ while (--i >= 0)
+ destroy_node(&test.nodes[i]);
+ free(test.nodes);
+ return ret;
+}
+
+static void destroy_nodes(void)
+{
+ int i;
+
+ for (i = 0; i < connections; i++)
+ destroy_node(&test.nodes[i]);
+ free(test.nodes);
+}
+
+static int create_messages(void)
+{
+ if (!message_size)
+ message_count = 0;
+
+ if (!message_count)
+ return 0;
+
+ test.mem = malloc(message_size);
+ if (!test.mem) {
+ printf("failed message allocation\n");
+ return -1;
+ }
+ test.mr = ibv_reg_mr(test.pd, test.mem, message_size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!test.mr) {
+ printf("failed to reg MR\n");
+ goto err;
+ }
+ return 0;
+err:
+ free(test.mem);
+ return -1;
+}
+
+static void destroy_messages(void)
+{
+ if (!message_count)
+ return;
+
+ ibv_dereg_mr(test.mr);
+ free(test.mem);
+}
+
+static int init(void)
+{
+ struct dlist *dev_list;
+ int ret;
+
+ test.connects_left = connections;
+ test.disconnects_left = connections;
+
+ dev_list = ibv_get_devices();
+ dlist_start(dev_list);
+ test.device = dlist_next(dev_list);
+ if (!test.device)
+ return -1;
+
+ test.verbs = ibv_open_device(test.device);
+ if (!test.verbs)
+ return -1;
+
+ test.pd = ibv_alloc_pd(test.verbs);
+ if (!test.pd) {
+ printf("failed to alloc PD\n");
+ return -1;
+ }
+ ret = create_messages();
+ if (ret) {
+ printf("unable to create test messages\n");
+ goto error1;
+ }
+ ret = create_nodes();
+ if (ret) {
+ printf("unable to create test nodes\n");
+ goto error2;
+ }
+ return 0;
+error2:
+ destroy_messages();
+error1:
+ ibv_dealloc_pd(test.pd);
+ return -1;
+}
+
+static void cleanup(void)
+{
+ destroy_nodes();
+ destroy_messages();
+ ibv_dealloc_pd(test.pd);
+}
+
+static int send_msgs(void)
+{
+ struct ibv_send_wr send_wr, *bad_send_wr;
+ struct ibv_sge sge;
+ int i, m, ret;
+
+ send_wr.next = NULL;
+ send_wr.sg_list = &sge;
+ send_wr.num_sge = 1;
+ send_wr.opcode = IBV_WR_SEND;
+ send_wr.send_flags = IBV_SEND_SIGNALED;
+ send_wr.wr_id = 0;
+
+ sge.addr = (uintptr_t) test.mem;
+ sge.length = message_size;
+ sge.lkey = test.mr->lkey;
+
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ for (m = 0; m < message_count; m++) {
+ ret = ibv_post_send(test.nodes[i].qp, &send_wr,
+ &bad_send_wr);
+ if (ret)
+ return ret;
+ }
+ }
+ return 0;
+}
+
+static int poll_cqs(void)
+{
+ struct ibv_wc wc[8];
+ int done, i, ret;
+
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ for (done = 0; done < message_count; done += ret) {
+ ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
+ if (ret < 0) {
+ printf("failed polling CQ: %d\n", ret);
+ return ret;
+ }
+ }
+ }
+ return 0;
+}
+
+static void connect_events(void)
+{
+ struct ib_cm_event *event;
+ int err = 0;
+
+ while (test.connects_left && !err) {
+ err = ib_cm_event_get(&event);
+ if (!err) {
+ cm_handler(event->cm_id, event);
+ ib_cm_event_put(event);
+ }
+ }
+}
+
+static void disconnect_events(void)
+{
+ struct ib_cm_event *event;
+ int err = 0;
+
+ while (test.disconnects_left && !err) {
+ err = ib_cm_event_get(&event);
+ if (!err) {
+ cm_handler(event->cm_id, event);
+ ib_cm_event_put(event);
+ }
+ }
+}
+
+static void run_server(void)
+{
+ struct ib_cm_id *listen_id;
+ int i, ret;
+
+ printf("starting server\n");
+ if (ib_cm_create_id(&listen_id, &test)) {
+ printf("listen request failed\n");
+ return;
+ }
+ ret = ib_cm_listen(listen_id, cpu_to_be64(0x1000), 0);
+ if (ret) {
+ printf("failure trying to listen: %d\n", ret);
+ goto out;
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("initiating data transfers\n");
+ if (send_msgs())
+ goto out;
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+ printf("data transfers complete\n");
+ }
+
+ printf("disconnecting\n");
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ test.nodes[i].connected = 0;
+ ib_cm_send_dreq(test.nodes[i].cm_id, NULL, 0);
+ }
+ disconnect_events();
+ printf("disconnected\n");
+out:
+ ib_cm_destroy_id(listen_id);
+}
+
+static void at_callback(uint64_t req_id, void *context, int rec_num)
+{
+}
+
+static int query_for_path(char *dest)
+{
+ struct ib_at_ib_route route;
+ struct ib_at_completion comp;
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dest, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ comp.fn = at_callback;
+ ret = ib_at_route_by_ip(((struct sockaddr_in *)res->ai_addr)->sin_addr.s_addr,
+ 0, 0, 0, &route, &comp, NULL);
+ if (ret < 0) {
+ printf("ib_at_route_by_ip failed: %d\n", ret);
+ goto out;
+ }
+
+ if (!ret) {
+ ret = ib_at_callback_get();
+ if (ret) {
+ printf("ib_at_callback_get failed: %d\n", ret);
+ goto out;
+ }
+ }
+
+ ret = ib_at_paths_by_route(&route, 0, &test.path_rec, 1, &comp, NULL);
+ if (ret < 0) {
+ printf("ib_at_paths_by_route failed: %d\n", ret);
+ goto out;
+ }
+
+ if (!ret) {
+ ret = ib_at_callback_get();
+ if (ret)
+ printf("ib_at_callback_get failed: %d\n", ret);
+ } else
+ ret = 0;
+
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
+static void run_client(char *dest)
+{
+ struct ib_cm_req_param req;
+ int i, ret;
+
+ printf("starting client\n");
+ ret = query_for_path(dest);
+ if (ret) {
+ printf("failed path record query: %d\n", ret);
+ return;
+ }
+
+ memset(&req, 0, sizeof req);
+ req.primary_path = &test.path_rec;
+ req.service_id = cpu_to_be64(0x1000);
+ req.responder_resources = 1;
+ req.initiator_depth = 1;
+ req.remote_cm_response_timeout = 20;
+ req.local_cm_response_timeout = 20;
+ req.retry_count = 5;
+ req.max_cm_retries = 5;
+
+ printf("connecting\n");
+ for (i = 0; i < connections; i++) {
+ req.qp_num = test.nodes[i].qp->qp_num;
+ req.qp_type = IBV_QPT_RC;
+ req.srq = (test.nodes[i].qp->srq != NULL);
+ req.starting_psn = test.nodes[i].qp->qp_num;
+ ret = ib_cm_send_req(test.nodes[i].cm_id, &req);
+ if (ret) {
+ printf("failure sending REQ: %d\n", ret);
+ return;
+ }
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+ printf("initiating data transfers\n");
+ if (send_msgs())
+ goto out;
+ printf("data transfers complete\n");
+ }
+out:
+ disconnect_events();
+}
+
+int main(int argc, char **argv)
+{
+ if (argc != 1 && argc != 2) {
+ printf("usage: %s [server_addr]\n", argv[0]);
+ exit(1);
+ }
+
+ is_server = (argc == 1);
+ if (init())
+ exit(1);
+
+ if (is_server)
+ run_server();
+ else
+ run_client(argv[1]);
+
+ printf("test complete\n");
+ cleanup();
+ return 0;
+}
#define TEST_SID 0x0000000ff0000000ULL
-static int cm_connect(uint32_t cm_id)
+static int cm_connect(struct ib_cm_id *cm_id)
{
struct ib_cm_req_param param;
struct ib_sa_path_rec sa;
src->global.subnet_prefix = cpu_to_be64(0xfe80000000000000ULL);
dst->global.subnet_prefix = cpu_to_be64(0xfe80000000000000ULL);
- src->global.interface_id = cpu_to_be64(0x0002c90200002179ULL);
- dst->global.interface_id = cpu_to_be64(0x0005ad000001296cULL);
+ src->global.interface_id = cpu_to_be64(0x0002c90107fc5e11ULL);
+ dst->global.interface_id = cpu_to_be64(0x0002c90107fc5eb1ULL);
return ib_cm_send_req(cm_id, ¶m);
}
{
struct ib_cm_event *event;
struct ib_cm_rep_param rep;
- int cm_id;
+ struct ib_cm_id *cm_id;
int result;
int param_c = 0;
exit(1);
}
- result = ib_cm_create_id(&cm_id);
- if (result < 0) {
+ result = ib_cm_create_id(&cm_id, NULL);
+ if (result) {
printf("Error creating CM ID <%d:%d>\n", result, errno);
goto done;
}
if (mode) {
result = cm_connect(cm_id);
if (result) {
- printf("Error <%d:%d> sending REQ <%d>\n",
- result, errno, cm_id);
+ printf("Error <%d:%d> sending REQ\n",
+ result, errno);
goto done;
}
}
else {
result = ib_cm_listen(cm_id, TEST_SID, 0);
if (result) {
- printf("Error <%d:%d> listening <%d>\n",
- result, errno, cm_id);
+ printf("Error <%d:%d> listening\n",
+ result, errno);
goto done;
}
}
goto done;
}
- printf("CM ID <%d> Event <%d>\n", event->cm_id, event->event);
+ printf("CM ID <%p> Event <%d>\n", event->cm_id, event->event);
switch (event->event) {
case IB_CM_REQ_RECEIVED:
done:
return 0;
}
-
IB_CM_SIDR_REP_INFO_LENGTH = 72
};
+struct ib_cm_id {
+ void *context;
+ uint32_t handle;
+};
+
struct ib_cm_req_event_param {
- uint32_t listen_id;
+ struct ib_cm_id *listen_id;
struct ib_sa_path_rec *primary_path;
struct ib_sa_path_rec *alternate_path;
};
struct ib_cm_sidr_req_event_param {
- uint32_t listen_id;
+ struct ib_cm_id *listen_id;
struct ib_device *device;
uint8_t port;
uint16_t pkey;
};
struct ib_cm_event {
- uint32_t cm_id;
+ struct ib_cm_id *cm_id;
enum ib_cm_event_type event;
union {
struct ib_cm_req_event_param req_rcvd;
* Communication identifiers are used to track connection states, service
* ID resolution requests, and listen requests.
*/
-int ib_cm_create_id(uint32_t *cm_id);
+int ib_cm_create_id(struct ib_cm_id **cm_id, void *context);
/**
* ib_cm_destroy_id - Destroy a connection identifier.
* @cm_id: Connection identifier to destroy.
*/
-int ib_cm_destroy_id(uint32_t cm_id);
+int ib_cm_destroy_id(struct ib_cm_id *cm_id);
struct ib_cm_attr_param {
uint64_t service_id;
*
* Not all parameters are valid during all connection states.
*/
-int ib_cm_attr_id(uint32_t cm_id,
+int ib_cm_attr_id(struct ib_cm_id *cm_id,
struct ib_cm_attr_param *param);
/**
* range of service IDs. If set to 0, the service ID is matched
* exactly.
*/
-int ib_cm_listen(uint32_t cm_id,
+int ib_cm_listen(struct ib_cm_id *cm_id,
uint64_t service_id,
uint64_t service_mask);
* @param: Connection request information needed to establish the
* connection.
*/
-int ib_cm_send_req(uint32_t cm_id,
+int ib_cm_send_req(struct ib_cm_id *cm_id,
struct ib_cm_req_param *param);
struct ib_cm_rep_param {
* @param: Connection reply information needed to establish the
* connection.
*/
-int ib_cm_send_rep(uint32_t cm_id,
+int ib_cm_send_rep(struct ib_cm_id *cm_id,
struct ib_cm_rep_param *param);
/**
* ready to use message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_rtu(uint32_t cm_id,
+int ib_cm_send_rtu(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len);
* disconnection request message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_dreq(uint32_t cm_id,
+int ib_cm_send_dreq(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len);
* disconnection reply message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_drep(uint32_t cm_id,
+int ib_cm_send_drep(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len);
* This routine should be invoked by users who receive messages on a
* connected QP before an RTU has been received.
*/
-int ib_cm_establish(uint32_t cm_id);
+int ib_cm_establish(struct ib_cm_id *cm_id);
/**
* ib_cm_send_rej - Sends a connection rejection message to the
* rejection message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_rej(uint32_t cm_id,
+int ib_cm_send_rej(struct ib_cm_id *cm_id,
enum ib_cm_rej_reason reason,
void *ari,
uint8_t ari_length,
* message receipt acknowledgement.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_mra(uint32_t cm_id,
+int ib_cm_send_mra(struct ib_cm_id *cm_id,
uint8_t service_timeout,
void *private_data,
uint8_t private_data_len);
* load alternate path message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_lap(uint32_t cm_id,
+int ib_cm_send_lap(struct ib_cm_id *cm_id,
struct ib_sa_path_rec *alternate_path,
void *private_data,
uint8_t private_data_len);
+/**
+ * ib_cm_init_qp_attr - Initializes the QP attributes for use in transitioning
+ * to a specified QP state.
+ * @cm_id: Communication identifier associated with the QP attributes to
+ * initialize.
+ * @qp_attr: On input, specifies the desired QP state. On output, the
+ * mandatory and desired optional attributes will be set in order to
+ * modify the QP to the specified state.
+ * @qp_attr_mask: The QP attribute mask that may be used to transition the
+ * QP to the specified state.
+ *
+ * Users must set the @qp_attr->qp_state to the desired QP state. This call
+ * will set all required attributes for the given transition, along with
+ * known optional attributes. Users may override the attributes returned from
+ * this call before calling ib_modify_qp.
+ */
+int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
+ struct ibv_qp_attr *qp_attr,
+ int *qp_attr_mask);
+
/**
* ib_cm_send_apr - Sends an alternate path response message in response to
* a load alternate path request.
* alternate path response message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_apr(uint32_t cm_id,
+int ib_cm_send_apr(struct ib_cm_id *cm_id,
enum ib_cm_apr_status status,
void *info,
uint8_t info_length,
* service ID resolution request.
* @param: Service ID resolution request information.
*/
-int ib_cm_send_sidr_req(uint32_t cm_id,
+int ib_cm_send_sidr_req(struct ib_cm_id *cm_id,
struct ib_cm_sidr_req_param *param);
struct ib_cm_sidr_rep_param {
* resolution request.
* @param: Service ID resolution reply information.
*/
-int ib_cm_send_sidr_rep(uint32_t cm_id,
+int ib_cm_send_sidr_rep(struct ib_cm_id *cm_id,
struct ib_cm_sidr_rep_param *param);
#endif /* CM_H */
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* drivers/infiniband/include/ib_user_cm.h
*/
-#define IB_USER_CM_ABI_VERSION 1
+#define IB_USER_CM_ABI_VERSION 2
enum {
IB_USER_CM_CMD_CREATE_ID,
IB_USER_CM_CMD_SEND_SIDR_REP,
IB_USER_CM_CMD_EVENT,
+ IB_USER_CM_CMD_INIT_QP_ATTR,
};
/*
* command ABI structures.
};
struct cm_abi_create_id {
+ __u64 uid;
__u64 response;
};
};
struct cm_abi_destroy_id {
+ __u64 response;
__u32 id;
};
+struct cm_abi_destroy_id_resp {
+ __u32 events_reported;
+};
+
struct cm_abi_attr_id {
__u64 response;
__u32 id;
__u32 remote_id;
};
+struct cm_abi_init_qp_attr {
+ __u64 response;
+ __u32 id;
+ __u32 qp_state;
+};
+
+struct cm_abi_ah_attr {
+ __u8 grh_dgid[16];
+ __u32 grh_flow_label;
+ __u16 dlid;
+ __u16 reserved;
+ __u8 grh_sgid_index;
+ __u8 grh_hop_limit;
+ __u8 grh_traffic_class;
+ __u8 sl;
+ __u8 src_path_bits;
+ __u8 static_rate;
+ __u8 is_global;
+ __u8 port_num;
+};
+
+struct cm_abi_init_qp_attr_resp {
+ __u32 qp_attr_mask;
+ __u32 qp_state;
+ __u32 cur_qp_state;
+ __u32 path_mtu;
+ __u32 path_mig_state;
+ __u32 qkey;
+ __u32 rq_psn;
+ __u32 sq_psn;
+ __u32 dest_qp_num;
+ __u32 qp_access_flags;
+
+ struct cm_abi_ah_attr ah_attr;
+ struct cm_abi_ah_attr alt_ah_attr;
+
+ /* ibv_qp_cap */
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
+
+ __u16 pkey_index;
+ __u16 alt_pkey_index;
+ __u8 en_sqd_async_notify;
+ __u8 sq_draining;
+ __u8 max_rd_atomic;
+ __u8 max_dest_rd_atomic;
+ __u8 min_rnr_timer;
+ __u8 port_num;
+ __u8 timeout;
+ __u8 retry_cnt;
+ __u8 rnr_retry;
+ __u8 alt_port_num;
+ __u8 alt_timeout;
+};
+
struct cm_abi_listen {
__u64 service_id;
__u64 service_mask;
};
struct cm_abi_rep {
+ __u64 uid;
__u64 data;
__u32 id;
__u32 qpn;
};
struct cm_abi_req_event_resp {
- __u32 listen_id;
/* device */
/* port */
struct cm_abi_path_rec primary_path;
};
struct cm_abi_sidr_req_event_resp {
- __u32 listen_id;
/* device */
/* port */
__u16 pkey;
#define CM_ABI_PRES_ALTERNATE 0x08
struct cm_abi_event_resp {
+ __u64 uid;
__u32 id;
__u32 event;
__u32 present;
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
#include <stdint.h>
#include <poll.h>
#include <unistd.h>
+#include <pthread.h>
#include <infiniband/cm.h>
#include <infiniband/cm_abi.h>
resp = alloca(sizeof(*resp)); \
if (!resp) \
return -ENOMEM; \
- cmd->response = (unsigned long)resp;\
+ cmd->response = (uintptr_t)resp;\
} while (0)
#define CM_CREATE_MSG_CMD(msg, cmd, type, size) \
memset(cmd, 0, sizeof(*cmd)); \
} while (0)
+struct cm_id_private {
+ struct ib_cm_id id;
+ int events_completed;
+ pthread_cond_t cond;
+ pthread_mutex_t mut;
+};
+
static int fd;
+#define container_of(ptr, type, field) \
+ ((type *) ((void *)ptr - offsetof(type, field)))
+
static void __attribute__((constructor)) ib_cm_init(void)
{
fd = open(IB_UCM_DEV_PATH, O_RDWR);
abi->preference = sa->preference;
}
-int ib_cm_create_id(uint32_t *cm_id)
+static void ib_cm_free_id(struct cm_id_private *cm_id_priv)
+{
+ pthread_cond_destroy(&cm_id_priv->cond);
+ pthread_mutex_destroy(&cm_id_priv->mut);
+ free(cm_id_priv);
+}
+
+static struct cm_id_private *ib_cm_alloc_id(void *context)
+{
+ struct cm_id_private *cm_id_priv;
+
+ cm_id_priv = malloc(sizeof *cm_id_priv);
+ if (!cm_id_priv)
+ return NULL;
+
+ memset(cm_id_priv, 0, sizeof *cm_id_priv);
+ cm_id_priv->id.context = context;
+ pthread_mutex_init(&cm_id_priv->mut, NULL);
+ if (pthread_cond_init(&cm_id_priv->cond, NULL))
+ goto err;
+
+ return cm_id_priv;
+
+err: ib_cm_free_id(cm_id_priv);
+ return NULL;
+}
+
+int ib_cm_create_id(struct ib_cm_id **cm_id, void *context)
{
struct cm_abi_create_id_resp *resp;
struct cm_abi_create_id *cmd;
+ struct cm_id_private *cm_id_priv;
void *msg;
int result;
int size;
- if (!cm_id)
- return -EINVAL;
+ cm_id_priv = ib_cm_alloc_id(context);
+ if (!cm_id_priv)
+ return -ENOMEM;
- CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_CREATE_ID, size);
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_CREATE_ID, size);
+ cmd->uid = (uintptr_t) cm_id_priv;
result = write(fd, msg, size);
if (result != size)
- return (result > 0) ? -ENODATA : result;
+ goto err;
- *cm_id = resp->id;
+ cm_id_priv->id.handle = resp->id;
+ *cm_id = &cm_id_priv->id;
return 0;
+
+err: ib_cm_free_id(cm_id_priv);
+ return result;
}
-int ib_cm_destroy_id(uint32_t cm_id)
+int ib_cm_destroy_id(struct ib_cm_id *cm_id)
{
+ struct cm_abi_destroy_id_resp *resp;
struct cm_abi_destroy_id *cmd;
+ struct cm_id_private *cm_id_priv;
void *msg;
int result;
int size;
- CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_DESTROY_ID, size);
-
- cmd->id = cm_id;
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_DESTROY_ID, size);
+ cmd->id = cm_id->handle;
result = write(fd, msg, size);
if (result != size)
return (result > 0) ? -ENODATA : result;
+ cm_id_priv = container_of(cm_id, struct cm_id_private, id);
+
+ pthread_mutex_lock(&cm_id_priv->mut);
+ while (cm_id_priv->events_completed < resp->events_reported)
+ pthread_cond_wait(&cm_id_priv->cond, &cm_id_priv->mut);
+ pthread_mutex_unlock(&cm_id_priv->mut);
+
+ ib_cm_free_id(cm_id_priv);
return 0;
}
-int ib_cm_attr_id(uint32_t cm_id, struct ib_cm_attr_param *param)
+int ib_cm_attr_id(struct ib_cm_id *cm_id, struct ib_cm_attr_param *param)
{
struct cm_abi_attr_id_resp *resp;
struct cm_abi_attr_id *cmd;
if (!param)
return -EINVAL;
- CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_ATTR_ID, size);
-
- cmd->id = cm_id;
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_ATTR_ID, size);
+ cmd->id = cm_id->handle;
result = write(fd, msg, size);
if (result != size)
param->service_mask = resp->service_mask;
param->local_id = resp->local_id;
param->remote_id = resp->remote_id;
+ return 0;
+}
+
+static void ib_cm_copy_ah_attr(struct ibv_ah_attr *dest_attr,
+ struct cm_abi_ah_attr *src_attr)
+{
+ memcpy(dest_attr->grh.dgid.raw, src_attr->grh_dgid,
+ sizeof dest_attr->grh.dgid);
+ dest_attr->grh.flow_label = src_attr->grh_flow_label;
+ dest_attr->grh.sgid_index = src_attr->grh_sgid_index;
+ dest_attr->grh.hop_limit = src_attr->grh_hop_limit;
+ dest_attr->grh.traffic_class = src_attr->grh_traffic_class;
+
+ dest_attr->dlid = src_attr->dlid;
+ dest_attr->sl = src_attr->sl;
+ dest_attr->src_path_bits = src_attr->src_path_bits;
+ dest_attr->static_rate = src_attr->static_rate;
+ dest_attr->is_global = src_attr->is_global;
+ dest_attr->port_num = src_attr->port_num;
+}
+
+static void ib_cm_copy_qp_attr(struct ibv_qp_attr *dest_attr,
+ struct cm_abi_init_qp_attr_resp *src_attr)
+{
+ dest_attr->cur_qp_state = src_attr->cur_qp_state;
+ dest_attr->path_mtu = src_attr->path_mtu;
+ dest_attr->path_mig_state = src_attr->path_mig_state;
+ dest_attr->qkey = src_attr->qkey;
+ dest_attr->rq_psn = src_attr->rq_psn;
+ dest_attr->sq_psn = src_attr->sq_psn;
+ dest_attr->dest_qp_num = src_attr->dest_qp_num;
+ dest_attr->qp_access_flags = src_attr->qp_access_flags;
+
+ dest_attr->cap.max_send_wr = src_attr->max_send_wr;
+ dest_attr->cap.max_recv_wr = src_attr->max_recv_wr;
+ dest_attr->cap.max_send_sge = src_attr->max_send_sge;
+ dest_attr->cap.max_recv_sge = src_attr->max_recv_sge;
+ dest_attr->cap.max_inline_data = src_attr->max_inline_data;
+
+ ib_cm_copy_ah_attr(&dest_attr->ah_attr, &src_attr->ah_attr);
+ ib_cm_copy_ah_attr(&dest_attr->alt_ah_attr, &src_attr->alt_ah_attr);
+
+ dest_attr->pkey_index = src_attr->pkey_index;
+ dest_attr->alt_pkey_index = src_attr->alt_pkey_index;
+ dest_attr->en_sqd_async_notify = src_attr->en_sqd_async_notify;
+ dest_attr->sq_draining = src_attr->sq_draining;
+ dest_attr->max_rd_atomic = src_attr->max_rd_atomic;
+ dest_attr->max_dest_rd_atomic = src_attr->max_dest_rd_atomic;
+ dest_attr->min_rnr_timer = src_attr->min_rnr_timer;
+ dest_attr->port_num = src_attr->port_num;
+ dest_attr->timeout = src_attr->timeout;
+ dest_attr->retry_cnt = src_attr->retry_cnt;
+ dest_attr->rnr_retry = src_attr->rnr_retry;
+ dest_attr->alt_port_num = src_attr->alt_port_num;
+ dest_attr->alt_timeout = src_attr->alt_timeout;
+}
+
+int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
+ struct ibv_qp_attr *qp_attr,
+ int *qp_attr_mask)
+{
+ struct cm_abi_init_qp_attr_resp *resp;
+ struct cm_abi_init_qp_attr *cmd;
+ void *msg;
+ int result;
+ int size;
+
+ if (!qp_attr || !qp_attr_mask)
+ return -EINVAL;
+
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_INIT_QP_ATTR, size);
+ cmd->id = cm_id->handle;
+ cmd->qp_state = qp_attr->qp_state;
+
+ result = write(fd, msg, size);
+ if (result != size)
+ return (result > 0) ? -ENODATA : result;
+
+ *qp_attr_mask = resp->qp_attr_mask;
+ ib_cm_copy_qp_attr(qp_attr, resp);
return 0;
}
-int ib_cm_listen(uint32_t cm_id,
+int ib_cm_listen(struct ib_cm_id *cm_id,
uint64_t service_id,
uint64_t service_mask)
{
int size;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_LISTEN, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->service_id = service_id;
cmd->service_mask = service_mask;
return 0;
}
-int ib_cm_send_req(uint32_t cm_id, struct ib_cm_req_param *param)
+int ib_cm_send_req(struct ib_cm_id *cm_id, struct ib_cm_req_param *param)
{
struct cm_abi_path_rec *p_path;
struct cm_abi_path_rec *a_path;
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_REQ, size);
-
- cmd->id = cm_id;
- cmd->qpn = param->qp_num;
- cmd->qp_type = param->qp_type;
- cmd->psn = param->starting_psn;
- cmd->sid = param->service_id;
-
+ cmd->id = cm_id->handle;
+ cmd->qpn = param->qp_num;
+ cmd->qp_type = param->qp_type;
+ cmd->psn = param->starting_psn;
+ cmd->sid = param->service_id;
cmd->peer_to_peer = param->peer_to_peer;
cmd->responder_resources = param->responder_resources;
cmd->initiator_depth = param->initiator_depth;
cmd->srq = param->srq;
if (param->primary_path) {
-
p_path = alloca(sizeof(*p_path));
if (!p_path)
return -ENOMEM;
cm_param_path_get(p_path, param->primary_path);
- cmd->primary_path = (unsigned long)p_path;
+ cmd->primary_path = (uintptr_t) p_path;
}
if (param->alternate_path) {
-
a_path = alloca(sizeof(*a_path));
if (!a_path)
return -ENOMEM;
cm_param_path_get(a_path, param->alternate_path);
- cmd->alternate_path = (unsigned long)a_path;
+ cmd->alternate_path = (uintptr_t) a_path;
}
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->len = param->private_data_len;
}
return 0;
}
-int ib_cm_send_rep(uint32_t cm_id, struct ib_cm_rep_param *param)
+int ib_cm_send_rep(struct ib_cm_id *cm_id, struct ib_cm_rep_param *param)
{
struct cm_abi_rep *cmd;
void *msg;
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_REP, size);
-
- cmd->id = cm_id;
- cmd->qpn = param->qp_num;
- cmd->psn = param->starting_psn;
-
+ cmd->uid = (uintptr_t) container_of(cm_id, struct cm_id_private, id);
+ cmd->id = cm_id->handle;
+ cmd->qpn = param->qp_num;
+ cmd->psn = param->starting_psn;
cmd->responder_resources = param->responder_resources;
cmd->initiator_depth = param->initiator_depth;
cmd->target_ack_delay = param->target_ack_delay;
cmd->srq = param->srq;
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->len = param->private_data_len;
}
return 0;
}
-static inline int cm_send_private_data(uint32_t cm_id,
+static inline int cm_send_private_data(struct ib_cm_id *cm_id,
uint32_t type,
void *private_data,
uint8_t private_data_len)
int size;
CM_CREATE_MSG_CMD(msg, cmd, type, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->len = private_data_len;
}
return 0;
}
-int ib_cm_send_rtu(uint32_t cm_id,
+int ib_cm_send_rtu(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len)
{
private_data, private_data_len);
}
-int ib_cm_send_dreq(uint32_t cm_id,
+int ib_cm_send_dreq(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len)
{
private_data, private_data_len);
}
-int ib_cm_send_drep(uint32_t cm_id,
+int ib_cm_send_drep(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len)
{
private_data, private_data_len);
}
-int ib_cm_establish(uint32_t cm_id)
+int ib_cm_establish(struct ib_cm_id *cm_id)
{
struct cm_abi_establish *cmd;
void *msg;
int result;
int size;
- CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_ESTABLISH, size);
-
- cmd->id = cm_id;
+ CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_ESTABLISH, size);
+ cmd->id = cm_id->handle;
result = write(fd, msg, size);
if (result != size)
return 0;
}
-static inline int cm_send_status(uint32_t cm_id,
+static inline int cm_send_status(struct ib_cm_id *cm_id,
uint32_t type,
int status,
void *info,
int size;
CM_CREATE_MSG_CMD(msg, cmd, type, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->status = status;
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->data_len = private_data_len;
}
if (info && info_length) {
-
- cmd->info = (unsigned long)info;
+ cmd->info = (uintptr_t) info;
cmd->info_len = info_length;
}
return 0;
}
-int ib_cm_send_rej(uint32_t cm_id,
+int ib_cm_send_rej(struct ib_cm_id *cm_id,
enum ib_cm_rej_reason reason,
void *ari,
uint8_t ari_length,
private_data, private_data_len);
}
-int ib_cm_send_apr(uint32_t cm_id,
+int ib_cm_send_apr(struct ib_cm_id *cm_id,
enum ib_cm_apr_status status,
void *info,
uint8_t info_length,
private_data, private_data_len);
}
-int ib_cm_send_mra(uint32_t cm_id,
+int ib_cm_send_mra(struct ib_cm_id *cm_id,
uint8_t service_timeout,
void *private_data,
uint8_t private_data_len)
int size;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_MRA, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->timeout = service_timeout;
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->len = private_data_len;
}
return 0;
}
-int ib_cm_send_lap(uint32_t cm_id,
+int ib_cm_send_lap(struct ib_cm_id *cm_id,
struct ib_sa_path_rec *alternate_path,
void *private_data,
uint8_t private_data_len)
int size;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_LAP, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
if (alternate_path) {
-
abi_path = alloca(sizeof(*abi_path));
if (!abi_path)
return -ENOMEM;
cm_param_path_get(abi_path, alternate_path);
- cmd->path = (unsigned long)abi_path;
+ cmd->path = (uintptr_t) abi_path;
}
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->len = private_data_len;
}
return 0;
}
-int ib_cm_send_sidr_req(uint32_t cm_id, struct ib_cm_sidr_req_param *param)
+int ib_cm_send_sidr_req(struct ib_cm_id *cm_id,
+ struct ib_cm_sidr_req_param *param)
{
struct cm_abi_path_rec *abi_path;
struct cm_abi_sidr_req *cmd;
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_SIDR_REQ, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->sid = param->service_id;
cmd->timeout = param->timeout_ms;
cmd->pkey = param->pkey;
cmd->max_cm_retries = param->max_cm_retries;
if (param->path) {
-
abi_path = alloca(sizeof(*abi_path));
if (!abi_path)
return -ENOMEM;
cm_param_path_get(abi_path, param->path);
- cmd->path = (unsigned long)abi_path;
+ cmd->path = (uintptr_t) abi_path;
}
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->len = param->private_data_len;
}
return 0;
}
-int ib_cm_send_sidr_rep(uint32_t cm_id, struct ib_cm_sidr_rep_param *param)
+int ib_cm_send_sidr_rep(struct ib_cm_id *cm_id,
+ struct ib_cm_sidr_rep_param *param)
{
struct cm_abi_sidr_rep *cmd;
void *msg;
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_SIDR_REP, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->qpn = param->qp_num;
cmd->qkey = param->qkey;
cmd->status = param->status;
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->data_len = param->private_data_len;
}
if (param->info && param->info_length) {
-
- cmd->info = (unsigned long)param->info;
+ cmd->info = (uintptr_t) param->info;
cmd->info_len = param->info_length;
}
if (!kpath || !upath)
return;
- memcpy(upath->dgid.raw, kpath->dgid, sizeof(union ibv_gid));
- memcpy(upath->sgid.raw, kpath->sgid, sizeof(union ibv_gid));
+ memcpy(upath->dgid.raw, kpath->dgid, sizeof upath->dgid);
+ memcpy(upath->sgid.raw, kpath->sgid, sizeof upath->sgid);
upath->dlid = kpath->dlid;
upath->slid = kpath->slid;
static void cm_event_req_get(struct ib_cm_req_event_param *ureq,
struct cm_abi_req_event_resp *kreq)
{
- ureq->listen_id = kreq->listen_id;
-
ureq->remote_ca_guid = kreq->remote_ca_guid;
ureq->remote_qkey = kreq->remote_qkey;
ureq->remote_qpn = kreq->remote_qpn;
urep->rnr_retry_count = krep->rnr_retry_count;
urep->srq = krep->srq;
}
-static void cm_event_rej_get(struct ib_cm_rej_event_param *urej,
- struct cm_abi_rej_event_resp *krej)
-{
- urej->reason = krej->reason;
-}
-
-static void cm_event_mra_get(struct ib_cm_mra_event_param *umra,
- struct cm_abi_mra_event_resp *kmra)
-{
- umra->service_timeout = kmra->timeout;
-}
-
-static void cm_event_lap_get(struct ib_cm_lap_event_param *ulap,
- struct cm_abi_lap_event_resp *klap)
-{
- cm_event_path_get(ulap->alternate_path, &klap->path);
-}
-
-static void cm_event_apr_get(struct ib_cm_apr_event_param *uapr,
- struct cm_abi_apr_event_resp *kapr)
-{
- uapr->ap_status = kapr->status;
-}
-
-static void cm_event_sidr_req_get(struct ib_cm_sidr_req_event_param *ureq,
- struct cm_abi_sidr_req_event_resp *kreq)
-{
- ureq->listen_id = kreq->listen_id;
- ureq->pkey = kreq->pkey;
-}
static void cm_event_sidr_rep_get(struct ib_cm_sidr_rep_event_param *urep,
struct cm_abi_sidr_rep_event_resp *krep)
int ib_cm_event_get(struct ib_cm_event **event)
{
+ struct cm_id_private *cm_id_priv;
struct cm_abi_cmd_hdr *hdr;
struct cm_abi_event_get *cmd;
struct cm_abi_event_resp *resp;
if (!resp)
return -ENOMEM;
- cmd->response = (unsigned long)resp;
+ cmd->response = (uintptr_t) resp;
cmd->data_len = (uint8_t)(~0U);
cmd->info_len = (uint8_t)(~0U);
goto done;
}
- cmd->data = (unsigned long)data;
- cmd->info = (unsigned long)info;
+ cmd->data = (uintptr_t) data;
+ cmd->info = (uintptr_t) info;
result = write(fd, msg, size);
if (result != size) {
result = -ENOMEM;
goto done;
}
-
memset(evt, 0, sizeof(*evt));
-
- evt->cm_id = resp->id;
+ evt->cm_id = (void *) (uintptr_t) resp->uid;
evt->event = resp->event;
if (resp->present & CM_ABI_PRES_PRIMARY) {
-
path_a = malloc(sizeof(*path_a));
if (!path_a) {
result = -ENOMEM;
}
if (resp->present & CM_ABI_PRES_ALTERNATE) {
-
path_b = malloc(sizeof(*path_b));
if (!path_b) {
result = -ENOMEM;
goto done;
}
}
-
- if (resp->present & CM_ABI_PRES_DATA) {
-
- evt->private_data = data;
- data = NULL;
- }
switch (evt->event) {
case IB_CM_REQ_RECEIVED:
-
+ evt->param.req_rcvd.listen_id = evt->cm_id;
+ cm_id_priv = ib_cm_alloc_id(evt->cm_id->context);
+ if (!cm_id_priv) {
+ result = -ENOMEM;
+ goto done;
+ }
+ cm_id_priv->id.handle = resp->id;
+ evt->cm_id = &cm_id_priv->id;
evt->param.req_rcvd.primary_path = path_a;
evt->param.req_rcvd.alternate_path = path_b;
path_a = NULL;
path_b = NULL;
-
cm_event_req_get(&evt->param.req_rcvd, &resp->u.req_resp);
break;
case IB_CM_REP_RECEIVED:
-
cm_event_rep_get(&evt->param.rep_rcvd, &resp->u.rep_resp);
break;
case IB_CM_MRA_RECEIVED:
-
- cm_event_mra_get(&evt->param.mra_rcvd, &resp->u.mra_resp);
+ evt->param.mra_rcvd.service_timeout = resp->u.mra_resp.timeout;
break;
case IB_CM_REJ_RECEIVED:
-
- cm_event_rej_get(&evt->param.rej_rcvd, &resp->u.rej_resp);
-
+ evt->param.rej_rcvd.reason = resp->u.rej_resp.reason;
evt->param.rej_rcvd.ari = info;
info = NULL;
-
break;
case IB_CM_LAP_RECEIVED:
-
evt->param.lap_rcvd.alternate_path = path_b;
path_b = NULL;
-
- cm_event_lap_get(&evt->param.lap_rcvd, &resp->u.lap_resp);
+ cm_event_path_get(evt->param.lap_rcvd.alternate_path,
+ &resp->u.lap_resp.path);
break;
case IB_CM_APR_RECEIVED:
-
- cm_event_apr_get(&evt->param.apr_rcvd, &resp->u.apr_resp);
-
+ evt->param.apr_rcvd.ap_status = resp->u.apr_resp.status;
evt->param.apr_rcvd.apr_info = info;
info = NULL;
-
break;
case IB_CM_SIDR_REQ_RECEIVED:
-
- cm_event_sidr_req_get(&evt->param.sidr_req_rcvd,
- &resp->u.sidr_req_resp);
+ evt->param.sidr_req_rcvd.listen_id = evt->cm_id;
+ cm_id_priv = ib_cm_alloc_id(evt->cm_id->context);
+ if (!cm_id_priv) {
+ result = -ENOMEM;
+ goto done;
+ }
+ cm_id_priv->id.handle = resp->id;
+ evt->cm_id = &cm_id_priv->id;
+ evt->param.sidr_req_rcvd.pkey = resp->u.sidr_req_resp.pkey;
break;
case IB_CM_SIDR_REP_RECEIVED:
-
cm_event_sidr_rep_get(&evt->param.sidr_rep_rcvd,
&resp->u.sidr_rep_resp);
-
evt->param.sidr_rep_rcvd.info = info;
info = NULL;
-
break;
default:
-
evt->param.send_status = resp->u.send_status;
break;
}
+ if (resp->present & CM_ABI_PRES_DATA) {
+ evt->private_data = data;
+ data = NULL;
+ }
+
*event = evt;
evt = NULL;
result = 0;
int ib_cm_event_put(struct ib_cm_event *event)
{
+ struct cm_id_private *cm_id_priv;
+
if (!event)
return -EINVAL;
if (event->private_data)
free(event->private_data);
+ cm_id_priv = container_of(event->cm_id, struct cm_id_private, id);
+
switch (event->event) {
case IB_CM_REQ_RECEIVED:
-
- if (event->param.req_rcvd.primary_path)
- free(event->param.req_rcvd.primary_path);
-
+ cm_id_priv = container_of(event->param.req_rcvd.listen_id,
+ struct cm_id_private, id);
+ free(event->param.req_rcvd.primary_path);
if (event->param.req_rcvd.alternate_path)
free(event->param.req_rcvd.alternate_path);
break;
case IB_CM_REJ_RECEIVED:
-
if (event->param.rej_rcvd.ari)
free(event->param.rej_rcvd.ari);
break;
case IB_CM_LAP_RECEIVED:
-
- if (event->param.lap_rcvd.alternate_path)
- free(event->param.lap_rcvd.alternate_path);
+ free(event->param.lap_rcvd.alternate_path);
break;
case IB_CM_APR_RECEIVED:
-
if (event->param.apr_rcvd.apr_info)
free(event->param.apr_rcvd.apr_info);
break;
+ case IB_CM_SIDR_REQ_RECEIVED:
+ cm_id_priv = container_of(event->param.sidr_req_rcvd.listen_id,
+ struct cm_id_private, id);
+ break;
case IB_CM_SIDR_REP_RECEIVED:
-
if (event->param.sidr_rep_rcvd.info)
free(event->param.sidr_rep_rcvd.info);
default:
break;
}
+ pthread_mutex_lock(&cm_id_priv->mut);
+ cm_id_priv->events_completed++;
+ pthread_cond_signal(&cm_id_priv->cond);
+ pthread_mutex_unlock(&cm_id_priv->mut);
+
free(event);
return 0;
}
projects / ~shefty / libibcm.git / commitdiff