From: Sean Hefty <sean.hefty@intel.com>
Date: Sun, 21 Aug 2005 22:30:08 +0000 (+0000)
Subject: r3143: - Add user specified context to all uCM events.  Users will not retrieve
X-Git-Tag: v1.0-rc1~33
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=efee396cdf05d8088a8d9d1a040d6bf8c307b17b;p=~shefty%2Flibibcm.git

r3143: - Add user specified context to all uCM events.  Users will not retrieve
  any events associated with the context after destroying the corresponding
  cm_id.
- Provide the ib_cm_init_qp_attr() call to userspace clients of the CM.
  This call may be used to set QP attributes properly before modifying the QP.
- Fixes some error handling syncrhonization and cleanup issues.
- Performs some minor code cleanup.
- Replaces the ucm_simple test program with a userspace version of cmpost.
- Updates DAPL to the new API.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
---

diff --git a/AUTHORS b/AUTHORS
index c79db95..a694ab7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -1 +1,2 @@
+Sean Hefty		<sean.hefty@intel.com>
 Libor Michalek		<libor@topspin.com>
diff --git a/Makefile.am b/Makefile.am
index c1161f0..5b8c062 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,9 +18,11 @@ endif
 src_libibcm_la_SOURCES = src/cm.c
 src_libibcm_la_LDFLAGS = -avoid-version -module $(ucm_version_script)
 
-bin_PROGRAMS = examples/ucm_simple
-examples_ucm_simple_SOURCES = examples/simple.c
-examples_ucm_simple_LDADD = $(top_builddir)/src/libibcm.la
+bin_PROGRAMS = examples/ucmpost
+examples_ucmpost_SOURCES = examples/cmpost.c
+examples_ucmpost_LDADD = $(top_builddir)/src/libibcm.la \
+			 $(libdir)/libibverbs.la \
+			 $(libdir)/libibat.la
 
 libibcmincludedir = $(includedir)/infiniband
 
diff --git a/examples/cmpost.c b/examples/cmpost.c
new file mode 100644
index 0000000..ace5718
--- /dev/null
+++ b/examples/cmpost.c
@@ -0,0 +1,718 @@
+/*
+ * Copyright (c) 2004, 2005 Intel Corporation.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <byteswap.h>
+
+#include <infiniband/cm.h>
+#include <infiniband/at.h>
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
+static inline uint32_t cpu_to_be32(uint32_t x) { return x; }
+#else
+static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
+static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
+#endif
+
+/*
+ * To execute:
+ * Server: ucmpost
+ * Client: ucmpost server
+ */
+
+struct cmtest {
+	struct ibv_device	*device;
+	struct ibv_context	*verbs;
+	struct ibv_pd		*pd;
+
+	/* cm info */
+	struct ib_sa_path_rec	path_rec;
+
+	struct cmtest_node	*nodes;
+	int			conn_index;
+	int			connects_left;
+	int			disconnects_left;
+
+	/* memory region info */
+	struct ibv_mr		*mr;
+	void			*mem;
+};
+
+static struct cmtest test;
+static int message_count = 10;
+static int message_size = 100;
+static int connections = 1;
+static int is_server = 1;
+
+struct cmtest_node {
+	int			id;
+	struct ibv_cq		*cq;
+	struct ibv_qp		*qp;
+	struct ib_cm_id		*cm_id;
+	int			connected;
+};
+
+static int post_recvs(struct cmtest_node *node)
+{
+	struct ibv_recv_wr recv_wr, *recv_failure;
+	struct ibv_sge sge;
+	int i, ret = 0;
+
+	if (!message_count)
+		return 0;
+
+	recv_wr.next = NULL;
+	recv_wr.sg_list = &sge;
+	recv_wr.num_sge = 1;
+	recv_wr.wr_id = (uintptr_t) node;
+
+	sge.length = message_size;
+	sge.lkey = test.mr->lkey;
+	sge.addr = (uintptr_t) test.mem;
+
+	for (i = 0; i < message_count && !ret; i++ ) {
+		ret = ibv_post_recv(node->qp, &recv_wr, &recv_failure);
+		if (ret) {
+			printf("failed to post receives: %d\n", ret);
+			break;
+		}
+	}
+	return ret;
+}
+
+static int modify_to_rtr(struct cmtest_node *node)
+{
+	struct ibv_qp_attr qp_attr;
+	int qp_attr_mask, ret;
+
+	qp_attr.qp_state = IBV_QPS_INIT;
+	ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+	if (ret) {
+		printf("failed to init QP attr for INIT: %d\n", ret);
+		return ret;
+	}
+	ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+	if (ret) {
+		printf("failed to modify QP to INIT: %d\n", ret);
+		return ret;
+	}
+	qp_attr.qp_state = IBV_QPS_RTR;
+	ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+	if (ret) {
+		printf("failed to init QP attr for RTR: %d\n", ret);
+		return ret;
+	}
+	qp_attr.rq_psn = node->qp->qp_num;
+	ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+	if (ret) {
+		printf("failed to modify QP to RTR: %d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+static int modify_to_rts(struct cmtest_node *node)
+{
+	struct ibv_qp_attr qp_attr;
+	int qp_attr_mask, ret;
+
+	qp_attr.qp_state = IBV_QPS_RTS;
+	ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+	if (ret) {
+		printf("failed to init QP attr for RTS: %d\n", ret);
+		return ret;
+	}
+	ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+	if (ret) {
+		printf("failed to modify QP to RTS: %d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+static void req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+	struct cmtest_node *node;
+	struct ib_cm_req_event_param *req;
+	struct ib_cm_rep_param rep;
+	int ret;
+
+	if (test.conn_index == connections)
+		goto error1;
+	node = &test.nodes[test.conn_index++];
+
+	node->cm_id = cm_id;
+	cm_id->context = node;
+
+	ret = modify_to_rtr(node);
+	if (ret)
+		goto error2;
+
+	ret = post_recvs(node);
+	if (ret)
+		goto error2;
+
+	req = &event->param.req_rcvd;
+	memset(&rep, 0, sizeof rep);
+	rep.qp_num = node->qp->qp_num;
+	rep.srq = (node->qp->srq != NULL);
+	rep.starting_psn = node->qp->qp_num;
+	rep.responder_resources = req->responder_resources;
+	rep.initiator_depth = req->initiator_depth;
+	rep.target_ack_delay = 20;
+	rep.flow_control = req->flow_control;
+	rep.rnr_retry_count = req->rnr_retry_count;
+
+	ret = ib_cm_send_rep(cm_id, &rep);
+	if (ret) {
+		printf("failed to send CM REP: %d\n", ret);
+		goto error2;
+	}
+	return;
+error2:
+	test.disconnects_left--;
+	test.connects_left--;
+error1:
+	printf("failing connection request\n");
+	ib_cm_send_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
+}
+
+static void rep_handler(struct cmtest_node *node, struct ib_cm_event *event)
+{
+	int ret;
+
+	ret = modify_to_rtr(node);
+	if (ret)
+		goto error;
+
+	ret = modify_to_rts(node);
+	if (ret)
+		goto error;
+
+	ret = post_recvs(node);
+	if (ret)
+		goto error;
+
+	ret = ib_cm_send_rtu(node->cm_id, NULL, 0);
+	if (ret) {
+		printf("failed to send CM RTU: %d\n", ret);
+		goto error;
+	}
+	node->connected = 1;
+	test.connects_left--;
+	return;
+error:
+	printf("failing connection reply\n");
+	ib_cm_send_rej(node->cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
+	test.disconnects_left--;
+	test.connects_left--;
+}
+
+static void rtu_handler(struct cmtest_node *node)
+{
+	int ret;
+
+	ret = modify_to_rts(node);
+	if (ret)
+		goto error;
+
+	node->connected = 1;
+	test.connects_left--;
+	return;
+error:
+	printf("aborting connection - disconnecting\n");
+	ib_cm_send_dreq(node->cm_id, NULL, 0);
+	test.disconnects_left--;
+	test.connects_left--;
+}
+
+static void cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+	struct cmtest_node *node = cm_id->context;
+
+	switch (event->event) {
+	case IB_CM_REQ_RECEIVED:
+		req_handler(cm_id, event);
+		break;
+	case IB_CM_REP_RECEIVED:
+		rep_handler(node, event);
+		break;
+	case IB_CM_RTU_RECEIVED:
+		rtu_handler(node);
+		break;
+	case IB_CM_DREQ_RECEIVED:
+		node->connected = 0;
+		ib_cm_send_drep(node->cm_id, NULL, 0);
+		test.disconnects_left--;
+		break;
+	case IB_CM_DREP_RECEIVED:
+		test.disconnects_left--;
+		break;
+	case IB_CM_REJ_RECEIVED:
+		printf("Received REJ\n");
+		/* fall through */
+	case IB_CM_REQ_ERROR:
+	case IB_CM_REP_ERROR:
+		printf("Error sending REQ or REP\n");
+		test.disconnects_left--;
+		test.connects_left--;
+		break;
+	case IB_CM_DREQ_ERROR:
+		test.disconnects_left--;
+		printf("Error sending DREQ\n");
+		break;
+	default:
+		break;
+	}
+}
+
+static int init_node(struct cmtest_node *node, struct ibv_qp_init_attr *qp_attr)
+{
+	int cqe, ret;
+
+	if (!is_server) {
+		ret = ib_cm_create_id(&node->cm_id, node);
+		if (ret) {
+			printf("failed to create cm_id: %d\n", ret);
+			return ret;
+		}
+	}
+
+	cqe = message_count ? message_count * 2 : 2;
+	node->cq = ibv_create_cq(test.verbs, cqe, node);
+	if (!node->cq) {
+		printf("unable to create CQ\n");
+		goto error1;
+	}
+
+	qp_attr->send_cq = node->cq; 
+	qp_attr->recv_cq = node->cq; 
+	node->qp = ibv_create_qp(test.pd, qp_attr);
+	if (!node->qp) {
+		printf("unable to create QP\n");
+		goto error2;
+	}
+	return 0;
+error2:
+	ibv_destroy_cq(node->cq);
+error1:
+	if (!is_server)
+		ib_cm_destroy_id(node->cm_id);
+	return -1;
+}
+
+static void destroy_node(struct cmtest_node *node)
+{
+	ibv_destroy_qp(node->qp);
+	ibv_destroy_cq(node->cq);
+	if (node->cm_id)
+		ib_cm_destroy_id(node->cm_id);
+}
+
+static int create_nodes(void)
+{
+	struct ibv_qp_init_attr qp_attr;
+	int ret, i;
+
+	test.nodes = malloc(sizeof *test.nodes * connections);
+	if (!test.nodes) {
+		printf("unable to allocate memory for test nodes\n");
+		return -1;
+	}
+	memset(test.nodes, 0, sizeof *test.nodes * connections);
+
+	memset(&qp_attr, 0, sizeof qp_attr);
+	qp_attr.cap.max_send_wr = message_count ? message_count : 1;
+	qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
+	qp_attr.cap.max_send_sge = 1;
+	qp_attr.cap.max_recv_sge = 1;
+	qp_attr.qp_type = IBV_QPT_RC;
+
+	for (i = 0; i < connections; i++) {
+		test.nodes[i].id = i;
+		ret = init_node(&test.nodes[i], &qp_attr);
+		if (ret)
+			goto error;
+	}
+	return 0;
+error:
+	while (--i >= 0)
+		destroy_node(&test.nodes[i]);
+	free(test.nodes);
+	return ret;
+}
+
+static void destroy_nodes(void)
+{
+	int i;
+
+	for (i = 0; i < connections; i++)
+		destroy_node(&test.nodes[i]);
+	free(test.nodes);
+}
+
+static int create_messages(void)
+{
+	if (!message_size)
+		message_count = 0;
+
+	if (!message_count)
+		return 0;
+
+	test.mem = malloc(message_size);
+	if (!test.mem) {
+		printf("failed message allocation\n");
+		return -1;
+	}
+	test.mr = ibv_reg_mr(test.pd, test.mem, message_size,
+			     IBV_ACCESS_LOCAL_WRITE);
+	if (!test.mr) {
+		printf("failed to reg MR\n");
+		goto err;
+	}
+	return 0;
+err:
+	free(test.mem);
+	return -1;
+}
+
+static void destroy_messages(void)
+{
+	if (!message_count)
+		return;
+
+	ibv_dereg_mr(test.mr);
+	free(test.mem);
+}
+
+static int init(void)
+{
+	struct dlist *dev_list;
+	int ret;
+
+	test.connects_left = connections;
+	test.disconnects_left = connections;
+
+	dev_list = ibv_get_devices();
+	dlist_start(dev_list);
+	test.device = dlist_next(dev_list);
+	if (!test.device)
+		return -1;
+
+	test.verbs = ibv_open_device(test.device);
+	if (!test.verbs)
+		return -1;
+
+	test.pd = ibv_alloc_pd(test.verbs);
+	if (!test.pd) {
+		printf("failed to alloc PD\n");
+		return -1;
+	}
+	ret = create_messages();
+	if (ret) {
+		printf("unable to create test messages\n");
+		goto error1;
+	}
+	ret = create_nodes();
+	if (ret) {
+		printf("unable to create test nodes\n");
+		goto error2;
+	}
+	return 0;
+error2:
+	destroy_messages();
+error1:
+	ibv_dealloc_pd(test.pd);
+	return -1;
+}
+
+static void cleanup(void)
+{
+	destroy_nodes();
+	destroy_messages();
+	ibv_dealloc_pd(test.pd);
+}
+
+static int send_msgs(void)
+{
+	struct ibv_send_wr send_wr, *bad_send_wr;
+	struct ibv_sge sge;
+	int i, m, ret;
+
+	send_wr.next = NULL;
+	send_wr.sg_list = &sge;
+	send_wr.num_sge = 1;
+	send_wr.opcode = IBV_WR_SEND;
+	send_wr.send_flags = IBV_SEND_SIGNALED;
+	send_wr.wr_id = 0;
+
+	sge.addr = (uintptr_t) test.mem;
+	sge.length = message_size;
+	sge.lkey = test.mr->lkey;
+
+	for (i = 0; i < connections; i++) {
+		if (!test.nodes[i].connected)
+			continue;
+
+		for (m = 0; m < message_count; m++) {
+			ret = ibv_post_send(test.nodes[i].qp, &send_wr,
+					    &bad_send_wr);
+			if (ret)
+				return ret;
+		}
+	}
+	return 0;
+}
+
+static int poll_cqs(void)
+{
+	struct ibv_wc wc[8];
+	int done, i, ret;
+
+	for (i = 0; i < connections; i++) {
+		if (!test.nodes[i].connected)
+			continue;
+
+		for (done = 0; done < message_count; done += ret) {
+			ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
+			if (ret < 0) {
+				printf("failed polling CQ: %d\n", ret);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
+static void connect_events(void)
+{
+	struct ib_cm_event *event;
+	int err = 0;
+
+	while (test.connects_left && !err) {
+		err = ib_cm_event_get(&event);
+		if (!err) {
+			cm_handler(event->cm_id, event);
+			ib_cm_event_put(event);
+		}
+	}
+}
+
+static void disconnect_events(void)
+{
+	struct ib_cm_event *event;
+	int err = 0;
+
+	while (test.disconnects_left && !err) {
+		err = ib_cm_event_get(&event);
+		if (!err) {
+			cm_handler(event->cm_id, event);
+			ib_cm_event_put(event);
+		}
+	}
+}
+
+static void run_server(void)
+{
+	struct ib_cm_id *listen_id;
+	int i, ret;
+
+	printf("starting server\n");
+	if (ib_cm_create_id(&listen_id, &test)) {
+		printf("listen request failed\n");
+		return;
+	}
+	ret = ib_cm_listen(listen_id, cpu_to_be64(0x1000), 0);
+	if (ret) {
+		printf("failure trying to listen: %d\n", ret);
+		goto out;
+	}
+
+	connect_events();
+
+	if (message_count) {
+		printf("initiating data transfers\n");
+		if (send_msgs())
+			goto out;
+		printf("receiving data transfers\n");
+		if (poll_cqs())
+			goto out;
+		printf("data transfers complete\n");
+	}
+
+	printf("disconnecting\n");
+	for (i = 0; i < connections; i++) {
+		if (!test.nodes[i].connected)
+			continue;
+
+		test.nodes[i].connected = 0;
+		ib_cm_send_dreq(test.nodes[i].cm_id, NULL, 0);
+	}
+	disconnect_events();
+ 	printf("disconnected\n");
+out:
+	ib_cm_destroy_id(listen_id);
+}
+
+static void at_callback(uint64_t req_id, void *context, int rec_num)
+{
+}
+
+static int query_for_path(char *dest)
+{
+	struct ib_at_ib_route route;
+	struct ib_at_completion comp;
+	struct addrinfo *res;
+	int ret;
+
+	ret = getaddrinfo(dest, NULL, NULL, &res);
+	if (ret) {
+		printf("getaddrinfo failed - invalid hostname or IP address\n");
+		return ret;
+	}
+
+	if (res->ai_family != PF_INET) {
+		ret = -1;
+		goto out;
+	}
+
+	comp.fn = at_callback;
+	ret = ib_at_route_by_ip(((struct sockaddr_in *)res->ai_addr)->sin_addr.s_addr,
+				0, 0, 0, &route, &comp, NULL);
+	if (ret < 0) {
+		printf("ib_at_route_by_ip failed: %d\n", ret);
+		goto out;
+	}
+
+	if (!ret) {
+		ret = ib_at_callback_get();
+		if (ret) {
+			printf("ib_at_callback_get failed: %d\n", ret);
+			goto out;
+		}
+	}
+
+	ret = ib_at_paths_by_route(&route, 0, &test.path_rec, 1, &comp, NULL);
+	if (ret < 0) {
+		printf("ib_at_paths_by_route failed: %d\n", ret);
+		goto out;
+	}
+
+	if (!ret) {
+		ret = ib_at_callback_get();
+		if (ret)
+			printf("ib_at_callback_get failed: %d\n", ret);
+	} else
+		ret = 0;
+
+out:
+	freeaddrinfo(res);
+	return ret;
+}
+
+static void run_client(char *dest)
+{
+	struct ib_cm_req_param req;
+	int i, ret;
+
+	printf("starting client\n");
+	ret = query_for_path(dest);
+	if (ret) {
+		printf("failed path record query: %d\n", ret);
+		return;
+	}
+
+	memset(&req, 0, sizeof req);
+	req.primary_path = &test.path_rec;
+	req.service_id = cpu_to_be64(0x1000);
+	req.responder_resources = 1;
+	req.initiator_depth = 1;
+	req.remote_cm_response_timeout = 20;
+	req.local_cm_response_timeout = 20;
+	req.retry_count = 5;
+	req.max_cm_retries = 5;
+
+	printf("connecting\n");
+	for (i = 0; i < connections; i++) {
+		req.qp_num = test.nodes[i].qp->qp_num;
+		req.qp_type = IBV_QPT_RC;
+		req.srq = (test.nodes[i].qp->srq != NULL);
+		req.starting_psn = test.nodes[i].qp->qp_num;
+		ret = ib_cm_send_req(test.nodes[i].cm_id, &req);
+		if (ret) {
+			printf("failure sending REQ: %d\n", ret);
+			return;
+		}
+	}
+
+	connect_events();
+
+	if (message_count) {
+		printf("receiving data transfers\n");
+		if (poll_cqs())
+			goto out;
+		printf("initiating data transfers\n");
+		if (send_msgs())
+			goto out;
+		printf("data transfers complete\n");
+	}
+out:
+	disconnect_events();
+}
+
+int main(int argc, char **argv)
+{
+	if (argc != 1 && argc != 2) {
+		printf("usage: %s [server_addr]\n", argv[0]);
+		exit(1);
+	}
+
+	is_server = (argc == 1);
+	if (init())
+		exit(1);
+
+	if (is_server)
+		run_server();
+	else
+		run_client(argv[1]);
+
+	printf("test complete\n");
+	cleanup();
+	return 0;
+}
diff --git a/examples/simple.c b/examples/simple.c
index 631d7b8..8cddbcb 100644
--- a/examples/simple.c
+++ b/examples/simple.c
@@ -58,7 +58,7 @@ static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
 
 #define TEST_SID 0x0000000ff0000000ULL
 
-static int cm_connect(uint32_t cm_id)
+static int cm_connect(struct ib_cm_id *cm_id)
 {
 	struct ib_cm_req_param param;
 	struct ib_sa_path_rec sa;
@@ -108,8 +108,8 @@ static int cm_connect(uint32_t cm_id)
 
 	src->global.subnet_prefix = cpu_to_be64(0xfe80000000000000ULL);
 	dst->global.subnet_prefix = cpu_to_be64(0xfe80000000000000ULL);
-	src->global.interface_id  = cpu_to_be64(0x0002c90200002179ULL);
-	dst->global.interface_id  = cpu_to_be64(0x0005ad000001296cULL);
+	src->global.interface_id  = cpu_to_be64(0x0002c90107fc5e11ULL);
+	dst->global.interface_id  = cpu_to_be64(0x0002c90107fc5eb1ULL);
 
 	return ib_cm_send_req(cm_id, &param);
 }
@@ -118,7 +118,7 @@ int main(int argc, char **argv)
 {
 	struct ib_cm_event *event;
 	struct ib_cm_rep_param rep;
-	int cm_id;
+	struct ib_cm_id *cm_id;
 	int result;
 
 	int param_c = 0;
@@ -137,8 +137,8 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	result = ib_cm_create_id(&cm_id);
-	if (result < 0) {
+	result = ib_cm_create_id(&cm_id, NULL);
+	if (result) {
 		printf("Error creating CM ID <%d:%d>\n", result, errno);
 		goto done;
 	}
@@ -146,16 +146,16 @@ int main(int argc, char **argv)
 	if (mode) {
 		result = cm_connect(cm_id);
 		if (result) {
-			printf("Error <%d:%d> sending REQ <%d>\n", 
-			       result, errno, cm_id);
+			printf("Error <%d:%d> sending REQ\n", 
+			       result, errno);
 			goto done;
 		}
 	}
 	else {
 		result = ib_cm_listen(cm_id, TEST_SID, 0);
 		if (result) {
-			printf("Error <%d:%d> listening <%d>\n", 
-			       result, errno, cm_id);
+			printf("Error <%d:%d> listening\n", 
+			       result, errno);
 			goto done;
 		}
 	}
@@ -169,7 +169,7 @@ int main(int argc, char **argv)
 			goto done;
 		}
 
-		printf("CM ID <%d> Event <%d>\n", event->cm_id, event->event);
+		printf("CM ID <%p> Event <%d>\n", event->cm_id, event->event);
 
 		switch (event->event) {
 		case IB_CM_REQ_RECEIVED:
@@ -264,4 +264,3 @@ int main(int argc, char **argv)
 done:
 	return 0;
 }
-
diff --git a/include/infiniband/cm.h b/include/infiniband/cm.h
index ab4eb88..8b278b4 100644
--- a/include/infiniband/cm.h
+++ b/include/infiniband/cm.h
@@ -77,8 +77,13 @@ enum ib_cm_data_size {
 	IB_CM_SIDR_REP_INFO_LENGTH	 = 72
 };
 
+struct ib_cm_id {
+	void			*context;
+	uint32_t		handle;
+};
+
 struct ib_cm_req_event_param {
-	uint32_t		listen_id;
+	struct ib_cm_id		*listen_id;
 
 	struct ib_sa_path_rec	*primary_path;
 	struct ib_sa_path_rec	*alternate_path;
@@ -187,7 +192,7 @@ struct ib_cm_apr_event_param {
 };
 
 struct ib_cm_sidr_req_event_param {
-	uint32_t	  listen_id;
+	struct ib_cm_id	 *listen_id;
 	struct ib_device *device;
 	uint8_t		  port;
 	uint16_t          pkey;
@@ -212,7 +217,7 @@ struct ib_cm_sidr_rep_event_param {
 };
 
 struct ib_cm_event {
-	uint32_t              cm_id;
+	struct ib_cm_id	      *cm_id;
 	enum ib_cm_event_type event;
 	union {
 		struct ib_cm_req_event_param	req_rcvd;
@@ -287,13 +292,13 @@ int ib_cm_get_fd(void);
  * Communication identifiers are used to track connection states, service
  * ID resolution requests, and listen requests.
  */
-int ib_cm_create_id(uint32_t *cm_id);
+int ib_cm_create_id(struct ib_cm_id **cm_id, void *context);
 
 /**
  * ib_cm_destroy_id - Destroy a connection identifier.
  * @cm_id: Connection identifier to destroy.
  */
-int ib_cm_destroy_id(uint32_t cm_id);
+int ib_cm_destroy_id(struct ib_cm_id *cm_id);
 
 struct ib_cm_attr_param {
 	uint64_t		service_id;
@@ -309,7 +314,7 @@ struct ib_cm_attr_param {
  *
  * Not all parameters are valid during all connection states.
  */
-int ib_cm_attr_id(uint32_t cm_id, 
+int ib_cm_attr_id(struct ib_cm_id *cm_id,
 		  struct ib_cm_attr_param *param);
 
 /**
@@ -323,7 +328,7 @@ int ib_cm_attr_id(uint32_t cm_id,
  *   range of service IDs.  If set to 0, the service ID is matched
  *   exactly.
  */
-int ib_cm_listen(uint32_t cm_id,
+int ib_cm_listen(struct ib_cm_id *cm_id,
 		 uint64_t service_id,
 		 uint64_t service_mask);
 
@@ -355,7 +360,7 @@ struct ib_cm_req_param {
  * @param: Connection request information needed to establish the
  *   connection.
  */
-int ib_cm_send_req(uint32_t cm_id,
+int ib_cm_send_req(struct ib_cm_id *cm_id,
 		   struct ib_cm_req_param *param);
 
 struct ib_cm_rep_param {
@@ -380,7 +385,7 @@ struct ib_cm_rep_param {
  * @param: Connection reply information needed to establish the
  *   connection.
  */
-int ib_cm_send_rep(uint32_t cm_id,
+int ib_cm_send_rep(struct ib_cm_id *cm_id,
 		   struct ib_cm_rep_param *param);
 
 /**
@@ -391,7 +396,7 @@ int ib_cm_send_rep(uint32_t cm_id,
  *   ready to use message.
  * @private_data_len: Size of the private data buffer, in bytes.
  */
-int ib_cm_send_rtu(uint32_t cm_id,
+int ib_cm_send_rtu(struct ib_cm_id *cm_id,
 		   void *private_data,
 		   uint8_t private_data_len);
 
@@ -404,7 +409,7 @@ int ib_cm_send_rtu(uint32_t cm_id,
  *   disconnection request message.
  * @private_data_len: Size of the private data buffer, in bytes.
  */
-int ib_cm_send_dreq(uint32_t cm_id,
+int ib_cm_send_dreq(struct ib_cm_id *cm_id,
 		    void *private_data,
 		    uint8_t private_data_len);
 
@@ -416,7 +421,7 @@ int ib_cm_send_dreq(uint32_t cm_id,
  *   disconnection reply message.
  * @private_data_len: Size of the private data buffer, in bytes.
  */
-int ib_cm_send_drep(uint32_t cm_id,
+int ib_cm_send_drep(struct ib_cm_id *cm_id,
 		    void *private_data,
 		    uint8_t private_data_len);
 
@@ -427,7 +432,7 @@ int ib_cm_send_drep(uint32_t cm_id,
  * This routine should be invoked by users who receive messages on a
  * connected QP before an RTU has been received.
  */
-int ib_cm_establish(uint32_t cm_id);
+int ib_cm_establish(struct ib_cm_id *cm_id);
 
 /**
  * ib_cm_send_rej - Sends a connection rejection message to the
@@ -441,7 +446,7 @@ int ib_cm_establish(uint32_t cm_id);
  *   rejection message.
  * @private_data_len: Size of the private data buffer, in bytes.
  */
-int ib_cm_send_rej(uint32_t cm_id,
+int ib_cm_send_rej(struct ib_cm_id *cm_id,
 		   enum ib_cm_rej_reason reason,
 		   void *ari,
 		   uint8_t ari_length,
@@ -458,7 +463,7 @@ int ib_cm_send_rej(uint32_t cm_id,
  *   message receipt acknowledgement.
  * @private_data_len: Size of the private data buffer, in bytes.
  */
-int ib_cm_send_mra(uint32_t cm_id,
+int ib_cm_send_mra(struct ib_cm_id *cm_id,
 		   uint8_t service_timeout,
 		   void *private_data,
 		   uint8_t private_data_len);
@@ -473,11 +478,31 @@ int ib_cm_send_mra(uint32_t cm_id,
  *   load alternate path message.
  * @private_data_len: Size of the private data buffer, in bytes.
  */
-int ib_cm_send_lap(uint32_t cm_id,
+int ib_cm_send_lap(struct ib_cm_id *cm_id,
 		   struct ib_sa_path_rec *alternate_path,
 		   void *private_data,
 		   uint8_t private_data_len);
 
+/**
+ * ib_cm_init_qp_attr - Initializes the QP attributes for use in transitioning
+ *   to a specified QP state.
+ * @cm_id: Communication identifier associated with the QP attributes to
+ *   initialize.
+ * @qp_attr: On input, specifies the desired QP state.  On output, the
+ *   mandatory and desired optional attributes will be set in order to
+ *   modify the QP to the specified state.
+ * @qp_attr_mask: The QP attribute mask that may be used to transition the
+ *   QP to the specified state.
+ *
+ * Users must set the @qp_attr->qp_state to the desired QP state.  This call
+ * will set all required attributes for the given transition, along with
+ * known optional attributes.  Users may override the attributes returned from
+ * this call before calling ib_modify_qp.
+ */
+int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
+		       struct ibv_qp_attr *qp_attr,
+		       int *qp_attr_mask);
+
 /**
  * ib_cm_send_apr - Sends an alternate path response message in response to
  *   a load alternate path request.
@@ -490,7 +515,7 @@ int ib_cm_send_lap(uint32_t cm_id,
  *   alternate path response message.
  * @private_data_len: Size of the private data buffer, in bytes.
  */
-int ib_cm_send_apr(uint32_t cm_id,
+int ib_cm_send_apr(struct ib_cm_id *cm_id,
 		   enum ib_cm_apr_status status,
 		   void *info,
 		   uint8_t info_length,
@@ -514,7 +539,7 @@ struct ib_cm_sidr_req_param {
  *   service ID resolution request.
  * @param: Service ID resolution request information.
  */
-int ib_cm_send_sidr_req(uint32_t cm_id,
+int ib_cm_send_sidr_req(struct ib_cm_id *cm_id,
 			struct ib_cm_sidr_req_param *param);
 
 struct ib_cm_sidr_rep_param {
@@ -534,7 +559,7 @@ struct ib_cm_sidr_rep_param {
  *   resolution request.
  * @param: Service ID resolution reply information.
  */
-int ib_cm_send_sidr_rep(uint32_t cm_id,
+int ib_cm_send_sidr_rep(struct ib_cm_id *cm_id,
 			struct ib_cm_sidr_rep_param *param);
 
 #endif /* CM_H */
diff --git a/include/infiniband/cm_abi.h b/include/infiniband/cm_abi.h
index c6e60fd..7d72329 100644
--- a/include/infiniband/cm_abi.h
+++ b/include/infiniband/cm_abi.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -41,7 +42,7 @@
  * drivers/infiniband/include/ib_user_cm.h
  */
 
-#define IB_USER_CM_ABI_VERSION 1
+#define IB_USER_CM_ABI_VERSION 2
 
 enum {
 	IB_USER_CM_CMD_CREATE_ID,
@@ -64,6 +65,7 @@ enum {
 	IB_USER_CM_CMD_SEND_SIDR_REP,
 
 	IB_USER_CM_CMD_EVENT,
+	IB_USER_CM_CMD_INIT_QP_ATTR,
 };
 /*
  * command ABI structures.
@@ -75,6 +77,7 @@ struct cm_abi_cmd_hdr {
 };
 
 struct cm_abi_create_id {
+	__u64 uid;
 	__u64 response;
 };
 
@@ -83,9 +86,14 @@ struct cm_abi_create_id_resp {
 };
 
 struct cm_abi_destroy_id {
+	__u64 response;
 	__u32 id;
 };
 
+struct cm_abi_destroy_id_resp {
+	__u32 events_reported;
+};
+
 struct cm_abi_attr_id {
 	__u64 response;
 	__u32 id;
@@ -98,6 +106,64 @@ struct cm_abi_attr_id_resp {
 	__u32 remote_id;
 };
 
+struct cm_abi_init_qp_attr {
+	__u64 response;
+	__u32 id;
+	__u32 qp_state;
+};
+
+struct cm_abi_ah_attr {
+	__u8	grh_dgid[16];
+	__u32	grh_flow_label;
+	__u16	dlid;
+	__u16	reserved;
+	__u8	grh_sgid_index;
+	__u8	grh_hop_limit;
+	__u8	grh_traffic_class;
+	__u8	sl;
+	__u8	src_path_bits;
+	__u8	static_rate;
+	__u8	is_global;
+	__u8	port_num;
+};
+
+struct cm_abi_init_qp_attr_resp {
+	__u32	qp_attr_mask;
+	__u32	qp_state;
+	__u32	cur_qp_state;
+	__u32	path_mtu;
+	__u32	path_mig_state;
+	__u32	qkey;
+	__u32	rq_psn;
+	__u32	sq_psn;
+	__u32	dest_qp_num;
+	__u32	qp_access_flags;
+
+	struct cm_abi_ah_attr	ah_attr;
+	struct cm_abi_ah_attr	alt_ah_attr;
+
+	/* ibv_qp_cap */
+	__u32	max_send_wr;
+	__u32	max_recv_wr;
+	__u32	max_send_sge;
+	__u32	max_recv_sge;
+	__u32	max_inline_data;
+
+	__u16	pkey_index;
+	__u16	alt_pkey_index;
+	__u8	en_sqd_async_notify;
+	__u8	sq_draining;
+	__u8	max_rd_atomic;
+	__u8	max_dest_rd_atomic;
+	__u8	min_rnr_timer;
+	__u8	port_num;
+	__u8	timeout;
+	__u8	retry_cnt;
+	__u8	rnr_retry;
+	__u8	alt_port_num;
+	__u8	alt_timeout;
+};
+
 struct cm_abi_listen {
 	__u64 service_id;
 	__u64 service_mask;
@@ -161,6 +227,7 @@ struct cm_abi_req {
 };
 
 struct cm_abi_rep {
+	__u64 uid;
 	__u64 data;
 	__u32 id;
 	__u32 qpn;
@@ -236,7 +303,6 @@ struct cm_abi_event_get {
 };
 
 struct cm_abi_req_event_resp {
-	__u32                  listen_id;
 	/* device */
 	/* port */
 	struct cm_abi_path_rec primary_path;
@@ -291,7 +357,6 @@ struct cm_abi_apr_event_resp {
 };
 
 struct cm_abi_sidr_req_event_resp {
-	__u32 listen_id;
 	/* device */
 	/* port */
 	__u16 pkey;
@@ -311,6 +376,7 @@ struct cm_abi_sidr_rep_event_resp {
 #define CM_ABI_PRES_ALTERNATE 0x08
 
 struct cm_abi_event_resp {
+	__u64 uid;
 	__u32 id;
 	__u32 event;
 	__u32 present;
diff --git a/src/cm.c b/src/cm.c
index 651c066..fea5bdb 100644
--- a/src/cm.c
+++ b/src/cm.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -45,6 +46,7 @@
 #include <stdint.h>
 #include <poll.h>
 #include <unistd.h>
+#include <pthread.h>
 
 #include <infiniband/cm.h>
 #include <infiniband/cm_abi.h>
@@ -69,7 +71,7 @@ do {                                        \
 	resp = alloca(sizeof(*resp));       \
 	if (!resp)                          \
 		return -ENOMEM;             \
-	cmd->response = (unsigned long)resp;\
+	cmd->response = (uintptr_t)resp;\
 } while (0)
 
 #define CM_CREATE_MSG_CMD(msg, cmd, type, size) \
@@ -88,8 +90,18 @@ do {                                        \
 	memset(cmd, 0, sizeof(*cmd));       \
 } while (0)
 
+struct cm_id_private {
+	struct ib_cm_id id;
+	int events_completed;
+	pthread_cond_t cond;
+	pthread_mutex_t mut;
+};
+
 static int fd;
 
+#define container_of(ptr, type, field) \
+	((type *) ((void *)ptr - offsetof(type, field)))
+
 static void __attribute__((constructor)) ib_cm_init(void)
 {
 	fd = open(IB_UCM_DEV_PATH, O_RDWR);
@@ -127,46 +139,89 @@ static void cm_param_path_get(struct cm_abi_path_rec *abi,
 	abi->preference                = sa->preference;
 }
 
-int ib_cm_create_id(uint32_t *cm_id)
+static void ib_cm_free_id(struct cm_id_private *cm_id_priv)
+{
+	pthread_cond_destroy(&cm_id_priv->cond);
+	pthread_mutex_destroy(&cm_id_priv->mut);
+	free(cm_id_priv);
+}
+
+static struct cm_id_private *ib_cm_alloc_id(void *context)
+{
+	struct cm_id_private *cm_id_priv;
+
+	cm_id_priv = malloc(sizeof *cm_id_priv);
+	if (!cm_id_priv)
+		return NULL;
+
+	memset(cm_id_priv, 0, sizeof *cm_id_priv);
+	cm_id_priv->id.context = context;
+	pthread_mutex_init(&cm_id_priv->mut, NULL);
+	if (pthread_cond_init(&cm_id_priv->cond, NULL))
+		goto err;
+
+	return cm_id_priv;
+
+err:	ib_cm_free_id(cm_id_priv);
+	return NULL;
+}
+
+int ib_cm_create_id(struct ib_cm_id **cm_id, void *context)
 {
 	struct cm_abi_create_id_resp *resp;
 	struct cm_abi_create_id *cmd;
+	struct cm_id_private *cm_id_priv;
 	void *msg;
 	int result;
 	int size;
 
-	if (!cm_id)
-		return -EINVAL;
+	cm_id_priv = ib_cm_alloc_id(context);
+	if (!cm_id_priv)
+		return -ENOMEM;
 
-        CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_CREATE_ID, size);
+	CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_CREATE_ID, size);
+	cmd->uid = (uintptr_t) cm_id_priv;
 
 	result = write(fd, msg, size);
 	if (result != size)
-		return (result > 0) ? -ENODATA : result;
+		goto err;
 
-	*cm_id = resp->id;
+	cm_id_priv->id.handle = resp->id;
+	*cm_id = &cm_id_priv->id;
 	return 0;
+
+err:	ib_cm_free_id(cm_id_priv);
+	return result;
 }
 
-int ib_cm_destroy_id(uint32_t cm_id)
+int ib_cm_destroy_id(struct ib_cm_id *cm_id)
 {
+	struct cm_abi_destroy_id_resp *resp;
 	struct cm_abi_destroy_id *cmd;
+	struct cm_id_private *cm_id_priv;
 	void *msg;
 	int result;
 	int size;
 	
-        CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_DESTROY_ID, size);
-
-	cmd->id = cm_id;
+	CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_DESTROY_ID, size);
+	cmd->id = cm_id->handle;
 
 	result = write(fd, msg, size);
 	if (result != size)
 		return (result > 0) ? -ENODATA : result;
 
+	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
+
+	pthread_mutex_lock(&cm_id_priv->mut);
+	while (cm_id_priv->events_completed < resp->events_reported)
+		pthread_cond_wait(&cm_id_priv->cond, &cm_id_priv->mut);
+	pthread_mutex_unlock(&cm_id_priv->mut);
+
+	ib_cm_free_id(cm_id_priv);
 	return 0;
 }
 
-int ib_cm_attr_id(uint32_t cm_id, struct ib_cm_attr_param *param)
+int ib_cm_attr_id(struct ib_cm_id *cm_id, struct ib_cm_attr_param *param)
 {
 	struct cm_abi_attr_id_resp *resp;
 	struct cm_abi_attr_id *cmd;
@@ -177,9 +232,8 @@ int ib_cm_attr_id(uint32_t cm_id, struct ib_cm_attr_param *param)
 	if (!param)
 		return -EINVAL;
 
-        CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_ATTR_ID, size);
-
-	cmd->id = cm_id;
+	CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_ATTR_ID, size);
+	cmd->id = cm_id->handle;
 
 	result = write(fd, msg, size);
 	if (result != size)
@@ -189,11 +243,91 @@ int ib_cm_attr_id(uint32_t cm_id, struct ib_cm_attr_param *param)
 	param->service_mask = resp->service_mask;
 	param->local_id     = resp->local_id;
 	param->remote_id    = resp->remote_id;
+	return 0;
+}
+
+static void ib_cm_copy_ah_attr(struct ibv_ah_attr *dest_attr,
+			       struct cm_abi_ah_attr *src_attr)
+{
+	memcpy(dest_attr->grh.dgid.raw, src_attr->grh_dgid,
+	       sizeof dest_attr->grh.dgid);
+	dest_attr->grh.flow_label = src_attr->grh_flow_label;
+	dest_attr->grh.sgid_index = src_attr->grh_sgid_index;
+	dest_attr->grh.hop_limit = src_attr->grh_hop_limit;
+	dest_attr->grh.traffic_class = src_attr->grh_traffic_class;
+
+	dest_attr->dlid = src_attr->dlid;
+	dest_attr->sl = src_attr->sl;
+	dest_attr->src_path_bits = src_attr->src_path_bits;
+	dest_attr->static_rate = src_attr->static_rate;
+	dest_attr->is_global = src_attr->is_global;
+	dest_attr->port_num = src_attr->port_num;
+}
+
+static void ib_cm_copy_qp_attr(struct ibv_qp_attr *dest_attr,
+			       struct cm_abi_init_qp_attr_resp *src_attr)
+{
+	dest_attr->cur_qp_state = src_attr->cur_qp_state;
+	dest_attr->path_mtu = src_attr->path_mtu;
+	dest_attr->path_mig_state = src_attr->path_mig_state;
+	dest_attr->qkey = src_attr->qkey;
+	dest_attr->rq_psn = src_attr->rq_psn;
+	dest_attr->sq_psn = src_attr->sq_psn;
+	dest_attr->dest_qp_num = src_attr->dest_qp_num;
+	dest_attr->qp_access_flags = src_attr->qp_access_flags;
+
+	dest_attr->cap.max_send_wr = src_attr->max_send_wr;
+	dest_attr->cap.max_recv_wr = src_attr->max_recv_wr;
+	dest_attr->cap.max_send_sge = src_attr->max_send_sge;
+	dest_attr->cap.max_recv_sge = src_attr->max_recv_sge;
+	dest_attr->cap.max_inline_data = src_attr->max_inline_data;
+
+	ib_cm_copy_ah_attr(&dest_attr->ah_attr, &src_attr->ah_attr);
+	ib_cm_copy_ah_attr(&dest_attr->alt_ah_attr, &src_attr->alt_ah_attr);
+
+	dest_attr->pkey_index = src_attr->pkey_index;
+	dest_attr->alt_pkey_index = src_attr->alt_pkey_index;
+	dest_attr->en_sqd_async_notify = src_attr->en_sqd_async_notify;
+	dest_attr->sq_draining = src_attr->sq_draining;
+	dest_attr->max_rd_atomic = src_attr->max_rd_atomic;
+	dest_attr->max_dest_rd_atomic = src_attr->max_dest_rd_atomic;
+	dest_attr->min_rnr_timer = src_attr->min_rnr_timer;
+	dest_attr->port_num = src_attr->port_num;
+	dest_attr->timeout = src_attr->timeout;
+	dest_attr->retry_cnt = src_attr->retry_cnt;
+	dest_attr->rnr_retry = src_attr->rnr_retry;
+	dest_attr->alt_port_num = src_attr->alt_port_num;
+	dest_attr->alt_timeout = src_attr->alt_timeout;
+}
+
+int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
+		       struct ibv_qp_attr *qp_attr,
+		       int *qp_attr_mask)
+{
+	struct cm_abi_init_qp_attr_resp *resp;
+	struct cm_abi_init_qp_attr *cmd;
+	void *msg;
+	int result;
+	int size;
+
+	if (!qp_attr || !qp_attr_mask)
+		return -EINVAL;
+
+	CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_INIT_QP_ATTR, size);
+	cmd->id = cm_id->handle;
+	cmd->qp_state = qp_attr->qp_state;
+
+	result = write(fd, msg, size);
+	if (result != size)
+		return (result > 0) ? -ENODATA : result;
+
+	*qp_attr_mask = resp->qp_attr_mask;
+	ib_cm_copy_qp_attr(qp_attr, resp);
 
 	return 0;
 }
 
-int ib_cm_listen(uint32_t cm_id,
+int ib_cm_listen(struct ib_cm_id *cm_id,
 		 uint64_t service_id,
 		 uint64_t service_mask)
 {
@@ -203,8 +337,7 @@ int ib_cm_listen(uint32_t cm_id,
 	int size;
 	
 	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_LISTEN, size);
-
-	cmd->id           = cm_id;
+	cmd->id           = cm_id->handle;
 	cmd->service_id   = service_id;
 	cmd->service_mask = service_mask;
 
@@ -215,7 +348,7 @@ int ib_cm_listen(uint32_t cm_id,
 	return 0;
 }
 
-int ib_cm_send_req(uint32_t cm_id, struct ib_cm_req_param *param)
+int ib_cm_send_req(struct ib_cm_id *cm_id, struct ib_cm_req_param *param)
 {
 	struct cm_abi_path_rec *p_path;
 	struct cm_abi_path_rec *a_path;
@@ -228,13 +361,11 @@ int ib_cm_send_req(uint32_t cm_id, struct ib_cm_req_param *param)
 		return -EINVAL;
 
 	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_REQ, size);
-
-	cmd->id      = cm_id;
-	cmd->qpn     = param->qp_num;
-	cmd->qp_type = param->qp_type;
-	cmd->psn     = param->starting_psn;
-        cmd->sid     = param->service_id;
-
+	cmd->id				= cm_id->handle;
+	cmd->qpn			= param->qp_num;
+	cmd->qp_type			= param->qp_type;
+	cmd->psn			= param->starting_psn;
+        cmd->sid			= param->service_id;
         cmd->peer_to_peer               = param->peer_to_peer;
         cmd->responder_resources        = param->responder_resources;
         cmd->initiator_depth            = param->initiator_depth;
@@ -247,28 +378,25 @@ int ib_cm_send_req(uint32_t cm_id, struct ib_cm_req_param *param)
         cmd->srq                        = param->srq;
 
 	if (param->primary_path) {
-
 		p_path = alloca(sizeof(*p_path));
 		if (!p_path)
 			return -ENOMEM;
 
 		cm_param_path_get(p_path, param->primary_path);
-		cmd->primary_path = (unsigned long)p_path;
+		cmd->primary_path = (uintptr_t) p_path;
 	}
 		
 	if (param->alternate_path) {
-
 		a_path = alloca(sizeof(*a_path));
 		if (!a_path)
 			return -ENOMEM;
 
 		cm_param_path_get(a_path, param->alternate_path);
-		cmd->alternate_path = (unsigned long)a_path;
+		cmd->alternate_path = (uintptr_t) a_path;
 	}
 
 	if (param->private_data && param->private_data_len) {
-
-		cmd->data = (unsigned long)param->private_data;
+		cmd->data = (uintptr_t) param->private_data;
 		cmd->len  = param->private_data_len;
 	}
 
@@ -279,7 +407,7 @@ int ib_cm_send_req(uint32_t cm_id, struct ib_cm_req_param *param)
 	return 0;
 }
 
-int ib_cm_send_rep(uint32_t cm_id, struct ib_cm_rep_param *param)
+int ib_cm_send_rep(struct ib_cm_id *cm_id, struct ib_cm_rep_param *param)
 {
 	struct cm_abi_rep *cmd;
 	void *msg;
@@ -290,11 +418,10 @@ int ib_cm_send_rep(uint32_t cm_id, struct ib_cm_rep_param *param)
 		return -EINVAL;
 
 	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_REP, size);
-
-	cmd->id  = cm_id;
-	cmd->qpn = param->qp_num;
-	cmd->psn = param->starting_psn;
-
+	cmd->uid = (uintptr_t) container_of(cm_id, struct cm_id_private, id);
+	cmd->id			 = cm_id->handle;
+	cmd->qpn		 = param->qp_num;
+	cmd->psn		 = param->starting_psn;
         cmd->responder_resources = param->responder_resources;
         cmd->initiator_depth     = param->initiator_depth;
 	cmd->target_ack_delay    = param->target_ack_delay;
@@ -304,8 +431,7 @@ int ib_cm_send_rep(uint32_t cm_id, struct ib_cm_rep_param *param)
         cmd->srq                 = param->srq;
 
 	if (param->private_data && param->private_data_len) {
-
-		cmd->data = (unsigned long)param->private_data;
+		cmd->data = (uintptr_t) param->private_data;
 		cmd->len  = param->private_data_len;
 	}
 
@@ -316,7 +442,7 @@ int ib_cm_send_rep(uint32_t cm_id, struct ib_cm_rep_param *param)
 	return 0;
 }
 
-static inline int cm_send_private_data(uint32_t cm_id,
+static inline int cm_send_private_data(struct ib_cm_id *cm_id,
 				       uint32_t type,
 				       void *private_data,
 				       uint8_t private_data_len)
@@ -327,12 +453,10 @@ static inline int cm_send_private_data(uint32_t cm_id,
 	int size;
 
 	CM_CREATE_MSG_CMD(msg, cmd, type, size);
-
-	cmd->id  = cm_id;
+	cmd->id = cm_id->handle;
 
 	if (private_data && private_data_len) {
-
-		cmd->data = (unsigned long)private_data;
+		cmd->data = (uintptr_t) private_data;
 		cmd->len  = private_data_len;
 	}
 
@@ -343,7 +467,7 @@ static inline int cm_send_private_data(uint32_t cm_id,
 	return 0;
 }
 
-int ib_cm_send_rtu(uint32_t cm_id,
+int ib_cm_send_rtu(struct ib_cm_id *cm_id,
 		   void *private_data,
 		   uint8_t private_data_len)
 {
@@ -351,7 +475,7 @@ int ib_cm_send_rtu(uint32_t cm_id,
 				    private_data, private_data_len);
 }
 
-int ib_cm_send_dreq(uint32_t cm_id,
+int ib_cm_send_dreq(struct ib_cm_id *cm_id,
 		    void *private_data,
 		    uint8_t private_data_len)
 {
@@ -359,7 +483,7 @@ int ib_cm_send_dreq(uint32_t cm_id,
 				    private_data, private_data_len);
 }
 
-int ib_cm_send_drep(uint32_t cm_id,
+int ib_cm_send_drep(struct ib_cm_id *cm_id,
 		    void *private_data,
 		    uint8_t private_data_len)
 {
@@ -367,16 +491,15 @@ int ib_cm_send_drep(uint32_t cm_id,
 				    private_data, private_data_len);
 }
 
-int ib_cm_establish(uint32_t cm_id)
+int ib_cm_establish(struct ib_cm_id *cm_id)
 {
 	struct cm_abi_establish *cmd;
 	void *msg;
 	int result;
 	int size;
 	
-        CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_ESTABLISH, size);
-
-	cmd->id = cm_id;
+	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_ESTABLISH, size);
+	cmd->id = cm_id->handle;
 
 	result = write(fd, msg, size);
 	if (result != size)
@@ -385,7 +508,7 @@ int ib_cm_establish(uint32_t cm_id)
 	return 0;
 }
 
-static inline int cm_send_status(uint32_t cm_id,
+static inline int cm_send_status(struct ib_cm_id *cm_id,
 				 uint32_t type,
 				 int status,
 				 void *info,
@@ -399,19 +522,16 @@ static inline int cm_send_status(uint32_t cm_id,
 	int size;
 
 	CM_CREATE_MSG_CMD(msg, cmd, type, size);
-
-	cmd->id     = cm_id;
+	cmd->id     = cm_id->handle;
 	cmd->status = status;
 
 	if (private_data && private_data_len) {
-
-		cmd->data     = (unsigned long)private_data;
+		cmd->data     = (uintptr_t) private_data;
 		cmd->data_len = private_data_len;
 	}
 
 	if (info && info_length) {
-
-		cmd->info     = (unsigned long)info;
+		cmd->info     = (uintptr_t) info;
 		cmd->info_len = info_length;
 	}
 
@@ -422,7 +542,7 @@ static inline int cm_send_status(uint32_t cm_id,
 	return 0;
 }
 
-int ib_cm_send_rej(uint32_t cm_id,
+int ib_cm_send_rej(struct ib_cm_id *cm_id,
 		   enum ib_cm_rej_reason reason,
 		   void *ari,
 		   uint8_t ari_length,
@@ -434,7 +554,7 @@ int ib_cm_send_rej(uint32_t cm_id,
 			      private_data, private_data_len);
 }
 
-int ib_cm_send_apr(uint32_t cm_id,
+int ib_cm_send_apr(struct ib_cm_id *cm_id,
 		   enum ib_cm_apr_status status,
 		   void *info,
 		   uint8_t info_length,
@@ -446,7 +566,7 @@ int ib_cm_send_apr(uint32_t cm_id,
 			      private_data, private_data_len);
 }
 
-int ib_cm_send_mra(uint32_t cm_id,
+int ib_cm_send_mra(struct ib_cm_id *cm_id,
 		   uint8_t service_timeout,
 		   void *private_data,
 		   uint8_t private_data_len)
@@ -457,13 +577,11 @@ int ib_cm_send_mra(uint32_t cm_id,
 	int size;
 
 	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_MRA, size);
-
-	cmd->id      = cm_id;
+	cmd->id      = cm_id->handle;
 	cmd->timeout = service_timeout;
 
 	if (private_data && private_data_len) {
-
-		cmd->data = (unsigned long)private_data;
+		cmd->data = (uintptr_t) private_data;
 		cmd->len  = private_data_len;
 	}
 
@@ -474,7 +592,7 @@ int ib_cm_send_mra(uint32_t cm_id,
 	return 0;
 }
 
-int ib_cm_send_lap(uint32_t cm_id,
+int ib_cm_send_lap(struct ib_cm_id *cm_id,
 		   struct ib_sa_path_rec *alternate_path,
 		   void *private_data,
 		   uint8_t private_data_len)
@@ -486,22 +604,19 @@ int ib_cm_send_lap(uint32_t cm_id,
 	int size;
 
 	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_LAP, size);
-
-	cmd->id = cm_id;
+	cmd->id = cm_id->handle;
 
 	if (alternate_path) {
-
 		abi_path = alloca(sizeof(*abi_path));
 		if (!abi_path)
 			return -ENOMEM;
 
 		cm_param_path_get(abi_path, alternate_path);
-		cmd->path = (unsigned long)abi_path;
+		cmd->path = (uintptr_t) abi_path;
 	}
 
 	if (private_data && private_data_len) {
-
-		cmd->data = (unsigned long)private_data;
+		cmd->data = (uintptr_t) private_data;
 		cmd->len  = private_data_len;
 	}
 
@@ -512,7 +627,8 @@ int ib_cm_send_lap(uint32_t cm_id,
 	return 0;
 }
 
-int ib_cm_send_sidr_req(uint32_t cm_id, struct ib_cm_sidr_req_param *param)
+int ib_cm_send_sidr_req(struct ib_cm_id *cm_id,
+			struct ib_cm_sidr_req_param *param)
 {
 	struct cm_abi_path_rec *abi_path;
 	struct cm_abi_sidr_req *cmd;
@@ -524,26 +640,23 @@ int ib_cm_send_sidr_req(uint32_t cm_id, struct ib_cm_sidr_req_param *param)
 		return -EINVAL;
 
 	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_SIDR_REQ, size);
-
-	cmd->id             = cm_id;
+	cmd->id             = cm_id->handle;
 	cmd->sid            = param->service_id;
 	cmd->timeout        = param->timeout_ms;
 	cmd->pkey           = param->pkey;
 	cmd->max_cm_retries = param->max_cm_retries;
 
 	if (param->path) {
-
 		abi_path = alloca(sizeof(*abi_path));
 		if (!abi_path)
 			return -ENOMEM;
 
 		cm_param_path_get(abi_path, param->path);
-		cmd->path = (unsigned long)abi_path;
+		cmd->path = (uintptr_t) abi_path;
 	}
 
 	if (param->private_data && param->private_data_len) {
-
-		cmd->data = (unsigned long)param->private_data;
+		cmd->data = (uintptr_t) param->private_data;
 		cmd->len  = param->private_data_len;
 	}
 
@@ -554,7 +667,8 @@ int ib_cm_send_sidr_req(uint32_t cm_id, struct ib_cm_sidr_req_param *param)
 	return 0;
 }
 
-int ib_cm_send_sidr_rep(uint32_t cm_id, struct ib_cm_sidr_rep_param *param)
+int ib_cm_send_sidr_rep(struct ib_cm_id *cm_id,
+			struct ib_cm_sidr_rep_param *param)
 {
 	struct cm_abi_sidr_rep *cmd;
 	void *msg;
@@ -565,21 +679,18 @@ int ib_cm_send_sidr_rep(uint32_t cm_id, struct ib_cm_sidr_rep_param *param)
 		return -EINVAL;
 
 	CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_SIDR_REP, size);
-
-	cmd->id     = cm_id;
+	cmd->id     = cm_id->handle;
 	cmd->qpn    = param->qp_num;
 	cmd->qkey   = param->qkey;
 	cmd->status = param->status;
 
 	if (param->private_data && param->private_data_len) {
-
-		cmd->data     = (unsigned long)param->private_data;
+		cmd->data     = (uintptr_t) param->private_data;
 		cmd->data_len = param->private_data_len;
 	}
 
 	if (param->info && param->info_length) {
-
-		cmd->info     = (unsigned long)param->info;
+		cmd->info     = (uintptr_t) param->info;
 		cmd->info_len = param->info_length;
 	}
 
@@ -599,8 +710,8 @@ static void cm_event_path_get(struct ib_sa_path_rec  *upath,
 	if (!kpath || !upath)
 		return;
 
-	memcpy(upath->dgid.raw, kpath->dgid, sizeof(union ibv_gid));
-	memcpy(upath->sgid.raw, kpath->sgid, sizeof(union ibv_gid));
+	memcpy(upath->dgid.raw, kpath->dgid, sizeof upath->dgid);
+	memcpy(upath->sgid.raw, kpath->sgid, sizeof upath->sgid);
 	
 	upath->dlid             = kpath->dlid;
 	upath->slid             = kpath->slid;
@@ -626,8 +737,6 @@ static void cm_event_path_get(struct ib_sa_path_rec  *upath,
 static void cm_event_req_get(struct ib_cm_req_event_param *ureq,
 			     struct cm_abi_req_event_resp *kreq)
 {
-	ureq->listen_id                  = kreq->listen_id;
-
 	ureq->remote_ca_guid             = kreq->remote_ca_guid;
 	ureq->remote_qkey                = kreq->remote_qkey;
 	ureq->remote_qpn                 = kreq->remote_qpn;
@@ -661,36 +770,6 @@ static void cm_event_rep_get(struct ib_cm_rep_event_param *urep,
 	urep->rnr_retry_count     = krep->rnr_retry_count;
 	urep->srq                 = krep->srq;
 }
-static void cm_event_rej_get(struct ib_cm_rej_event_param *urej,
-			     struct cm_abi_rej_event_resp *krej)
-{
-	urej->reason = krej->reason;
-}
-
-static void cm_event_mra_get(struct ib_cm_mra_event_param *umra,
-			     struct cm_abi_mra_event_resp *kmra)
-{
-	umra->service_timeout = kmra->timeout;
-}
-
-static void cm_event_lap_get(struct ib_cm_lap_event_param *ulap,
-			     struct cm_abi_lap_event_resp *klap)
-{
-	cm_event_path_get(ulap->alternate_path, &klap->path);
-}
-
-static void cm_event_apr_get(struct ib_cm_apr_event_param *uapr,
-			     struct cm_abi_apr_event_resp *kapr)
-{
-	uapr->ap_status = kapr->status;
-}
-
-static void cm_event_sidr_req_get(struct ib_cm_sidr_req_event_param *ureq,
-				  struct cm_abi_sidr_req_event_resp *kreq)
-{
-	ureq->listen_id = kreq->listen_id;
-	ureq->pkey      = kreq->pkey;
-}
 
 static void cm_event_sidr_rep_get(struct ib_cm_sidr_rep_event_param *urep,
 				  struct cm_abi_sidr_rep_event_resp *krep)
@@ -702,6 +781,7 @@ static void cm_event_sidr_rep_get(struct ib_cm_sidr_rep_event_param *urep,
 
 int ib_cm_event_get(struct ib_cm_event **event)
 {
+	struct cm_id_private *cm_id_priv;
 	struct cm_abi_cmd_hdr *hdr;
 	struct cm_abi_event_get *cmd;
 	struct cm_abi_event_resp *resp;
@@ -733,7 +813,7 @@ int ib_cm_event_get(struct ib_cm_event **event)
 	if (!resp)
 		return -ENOMEM;
 	
-	cmd->response = (unsigned long)resp;
+	cmd->response = (uintptr_t) resp;
 	cmd->data_len = (uint8_t)(~0U);
 	cmd->info_len = (uint8_t)(~0U);
 
@@ -749,8 +829,8 @@ int ib_cm_event_get(struct ib_cm_event **event)
 		goto done;
 	}
 
-	cmd->data = (unsigned long)data;
-	cmd->info = (unsigned long)info;
+	cmd->data = (uintptr_t) data;
+	cmd->info = (uintptr_t) info;
 
 	result = write(fd, msg, size);
 	if (result != size) {
@@ -765,14 +845,11 @@ int ib_cm_event_get(struct ib_cm_event **event)
 		result = -ENOMEM;
 		goto done;
 	}
-	
 	memset(evt, 0, sizeof(*evt));
-
-	evt->cm_id = resp->id;
+	evt->cm_id = (void *) (uintptr_t) resp->uid;
 	evt->event = resp->event;
 
 	if (resp->present & CM_ABI_PRES_PRIMARY) {
-
 		path_a = malloc(sizeof(*path_a));
 		if (!path_a) {
 			result = -ENOMEM;
@@ -781,81 +858,78 @@ int ib_cm_event_get(struct ib_cm_event **event)
 	}
 
 	if (resp->present & CM_ABI_PRES_ALTERNATE) {
-
 		path_b = malloc(sizeof(*path_b));
 		if (!path_b) {
 			result = -ENOMEM;
 			goto done;
 		}
 	}
-	
-	if (resp->present & CM_ABI_PRES_DATA) {
-
-		evt->private_data = data;
-		data = NULL;
-	}
 
 	switch (evt->event) {
 	case IB_CM_REQ_RECEIVED:
-
+		evt->param.req_rcvd.listen_id = evt->cm_id;
+		cm_id_priv = ib_cm_alloc_id(evt->cm_id->context);
+		if (!cm_id_priv) {
+			result = -ENOMEM;
+			goto done;
+		}
+		cm_id_priv->id.handle = resp->id;
+		evt->cm_id = &cm_id_priv->id;
 		evt->param.req_rcvd.primary_path   = path_a;
 		evt->param.req_rcvd.alternate_path = path_b;
 		path_a = NULL;
 		path_b = NULL;
-
 		cm_event_req_get(&evt->param.req_rcvd, &resp->u.req_resp);
 		break;
 	case IB_CM_REP_RECEIVED:
-
 		cm_event_rep_get(&evt->param.rep_rcvd, &resp->u.rep_resp);
 		break;
 	case IB_CM_MRA_RECEIVED:
-
-		cm_event_mra_get(&evt->param.mra_rcvd, &resp->u.mra_resp);
+		evt->param.mra_rcvd.service_timeout = resp->u.mra_resp.timeout;
 		break;
 	case IB_CM_REJ_RECEIVED:
-
-		cm_event_rej_get(&evt->param.rej_rcvd, &resp->u.rej_resp);
-
+		evt->param.rej_rcvd.reason = resp->u.rej_resp.reason;
 		evt->param.rej_rcvd.ari = info;
 		info = NULL;
-
 		break;
 	case IB_CM_LAP_RECEIVED:
-
 		evt->param.lap_rcvd.alternate_path = path_b;
 		path_b = NULL;
-
-		cm_event_lap_get(&evt->param.lap_rcvd, &resp->u.lap_resp);
+		cm_event_path_get(evt->param.lap_rcvd.alternate_path,
+				  &resp->u.lap_resp.path);
 		break;
 	case IB_CM_APR_RECEIVED:
-
-		cm_event_apr_get(&evt->param.apr_rcvd, &resp->u.apr_resp);
-
+		evt->param.apr_rcvd.ap_status = resp->u.apr_resp.status;
 		evt->param.apr_rcvd.apr_info = info;
 		info = NULL;
-
 		break;
 	case IB_CM_SIDR_REQ_RECEIVED:
-
-		cm_event_sidr_req_get(&evt->param.sidr_req_rcvd,
-				      &resp->u.sidr_req_resp);
+		evt->param.sidr_req_rcvd.listen_id = evt->cm_id;
+		cm_id_priv = ib_cm_alloc_id(evt->cm_id->context);
+		if (!cm_id_priv) {
+			result = -ENOMEM;
+			goto done;
+		}
+		cm_id_priv->id.handle = resp->id;
+		evt->cm_id = &cm_id_priv->id;
+		evt->param.sidr_req_rcvd.pkey = resp->u.sidr_req_resp.pkey;
 		break;
 	case IB_CM_SIDR_REP_RECEIVED:
-
 		cm_event_sidr_rep_get(&evt->param.sidr_rep_rcvd,
 				      &resp->u.sidr_rep_resp);
-		
 		evt->param.sidr_rep_rcvd.info = info;
 		info = NULL;
-
 		break;
 	default:
-
 		evt->param.send_status = resp->u.send_status;
 		break;
 	}
 
+	if (resp->present & CM_ABI_PRES_DATA) {
+		evt->private_data = data;
+		data = NULL;
+	}
+
 	*event = evt;
 	evt    = NULL;
 	result = 0;
@@ -876,44 +950,51 @@ done:
 
 int ib_cm_event_put(struct ib_cm_event *event)
 {
+	struct cm_id_private *cm_id_priv;
+
 	if (!event)
 		return -EINVAL;
 
 	if (event->private_data)
 		free(event->private_data);
 
+	cm_id_priv = container_of(event->cm_id, struct cm_id_private, id);
+
 	switch (event->event) {
 	case IB_CM_REQ_RECEIVED:
-
-		if (event->param.req_rcvd.primary_path)
-			free(event->param.req_rcvd.primary_path);
-
+		cm_id_priv = container_of(event->param.req_rcvd.listen_id,
+					  struct cm_id_private, id);
+		free(event->param.req_rcvd.primary_path);
 		if (event->param.req_rcvd.alternate_path)
 			free(event->param.req_rcvd.alternate_path);
 		break;
 	case IB_CM_REJ_RECEIVED:
-
 		if (event->param.rej_rcvd.ari)
 			free(event->param.rej_rcvd.ari);
 		break;
 	case IB_CM_LAP_RECEIVED:
-
-		if (event->param.lap_rcvd.alternate_path)
-			free(event->param.lap_rcvd.alternate_path);
+		free(event->param.lap_rcvd.alternate_path);
 		break;
 	case IB_CM_APR_RECEIVED:
-
 		if (event->param.apr_rcvd.apr_info)
 			free(event->param.apr_rcvd.apr_info);
 		break;
+	case IB_CM_SIDR_REQ_RECEIVED:
+		cm_id_priv = container_of(event->param.sidr_req_rcvd.listen_id,
+					  struct cm_id_private, id);
+		break;
 	case IB_CM_SIDR_REP_RECEIVED:
-
 		if (event->param.sidr_rep_rcvd.info)
 			free(event->param.sidr_rep_rcvd.info);
 	default:
 		break;
 	}
 
+	pthread_mutex_lock(&cm_id_priv->mut);
+	cm_id_priv->events_completed++;
+	pthread_cond_signal(&cm_id_priv->cond);
+	pthread_mutex_unlock(&cm_id_priv->mut);
+
 	free(event);
 	return 0;
 }