--- /dev/null
+Bottom: 7b3aa2b21ba60be16914f41821f67e5c9e95a8e3
+Top: 743c5750c90cd7c3f85b30f5b5d3a11738b2113c
+Author: Jay Sternberg <jay.e.sternberg@intel.com>
+Date: 2011-12-22 00:09:12 -0800
+
+libibverbs: Add xrc pingpong example
+
+Signed-off-by: Jay Sternberg <jay.e.sternberg@intel.com>
+Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+
+---
+
+diff --git a/Makefile.am b/Makefile.am
+index 0552e89..ee86c70 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -18,7 +18,7 @@ src_libibverbs_la_DEPENDENCIES = $(srcdir)/src/libibverbs.map
+
+ bin_PROGRAMS = examples/ibv_devices examples/ibv_devinfo \
+ examples/ibv_asyncwatch examples/ibv_rc_pingpong examples/ibv_uc_pingpong \
+- examples/ibv_ud_pingpong examples/ibv_srq_pingpong
++ examples/ibv_ud_pingpong examples/ibv_srq_pingpong examples/ibv_xsrq_pingpong
+ examples_ibv_devices_SOURCES = examples/device_list.c
+ examples_ibv_devices_LDADD = $(top_builddir)/src/libibverbs.la
+ examples_ibv_devinfo_SOURCES = examples/devinfo.c
+@@ -31,6 +31,8 @@ examples_ibv_ud_pingpong_SOURCES = examples/ud_pingpong.c examples/pingpong.c
+ examples_ibv_ud_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
+ examples_ibv_srq_pingpong_SOURCES = examples/srq_pingpong.c examples/pingpong.c
+ examples_ibv_srq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
++examples_ibv_xsrq_pingpong_SOURCES = examples/xsrq_pingpong.c examples/pingpong.c
++examples_ibv_xsrq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
+ examples_ibv_asyncwatch_SOURCES = examples/asyncwatch.c
+ examples_ibv_asyncwatch_LDADD = $(top_builddir)/src/libibverbs.la
+
+diff --git a/examples/xsrq_pingpong.c b/examples/xsrq_pingpong.c
+new file mode 100644
+index 0000000..c18b636
+--- /dev/null
++++ b/examples/xsrq_pingpong.c
+@@ -0,0 +1,873 @@
++/*\r
++ * Copyright (c) 2005 Topspin Communications. All rights reserved.\r
++ * Copyright (c) 2011 Intel Corporation, Inc. All rights reserved.\r
++ *\r
++ * This software is available to you under a choice of one of two\r
++ * licenses. You may choose to be licensed under the terms of the GNU\r
++ * General Public License (GPL) Version 2, available from the file\r
++ * COPYING in the main directory of this source tree, or the\r
++ * OpenIB.org BSD license below:\r
++ *\r
++ * Redistribution and use in source and binary forms, with or\r
++ * without modification, are permitted provided that the following\r
++ * conditions are met:\r
++ *\r
++ * - Redistributions of source code must retain the above\r
++ * copyright notice, this list of conditions and the following\r
++ * disclaimer.\r
++ *\r
++ * - Redistributions in binary form must reproduce the above\r
++ * copyright notice, this list of conditions and the following\r
++ * disclaimer in the documentation and/or other materials\r
++ * provided with the distribution.\r
++ *\r
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
++ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
++ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
++ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
++ * SOFTWARE.\r
++ */\r
++\r
++#if HAVE_CONFIG_H\r
++# include <config.h>\r
++#endif /* HAVE_CONFIG_H */\r
++\r
++#include <stdio.h>\r
++#include <fcntl.h>\r
++#include <errno.h>\r
++#include <stdlib.h>\r
++#include <unistd.h>\r
++#include <string.h>\r
++#include <sys/types.h>\r
++#include <sys/socket.h>\r
++#include <sys/time.h>\r
++#include <netdb.h>\r
++#include <malloc.h>\r
++#include <getopt.h>\r
++#include <arpa/inet.h>\r
++#include <time.h>\r
++\r
++#include "pingpong.h"\r
++\r
++#define MSG_FORMAT "%04x:%06x:%06x:%06x:%04x"\r
++#define MSG_SIZE 30\r
++#define MSG_SSCAN "%x:%x:%x:%x:%x"\r
++#define ADDR_FORMAT \\r
++ "%8s: LID %04x, QPN RECV %06x SEND %06x, PSN %06x, SRQN %04x\n"\r
++static int page_size;\r
++\r
++struct pingpong_dest {\r
++ int lid;\r
++ int recv_qpn;\r
++ int send_qpn;\r
++ int recv_psn;\r
++ int send_psn;\r
++ int srqn;\r
++ int pp_cnt;\r
++};\r
++\r
++struct pingpong_context {\r
++ struct ibv_context *context;\r
++ struct ibv_comp_channel *channel;\r
++ struct ibv_pd *pd;\r
++ struct ibv_mr *mr;\r
++ struct ibv_cq *send_cq;\r
++ struct ibv_cq *recv_cq;\r
++ struct ibv_srq *srq;\r
++ struct ibv_xrcd *xrcd;\r
++ struct ibv_qp **recv_qp;\r
++ struct ibv_qp **send_qp;\r
++ struct pingpong_dest *rem_dest;\r
++ void *buf;\r
++ int lid;\r
++ int sl;\r
++ enum ibv_mtu mtu;\r
++ int ib_port;\r
++ int fd;\r
++ int size;\r
++ int num_clients;\r
++ int num_tests;\r
++ int use_event;\r
++};\r
++\r
++struct pingpong_context ctx;\r
++\r
++\r
++static int open_device(char *ib_devname)\r
++{\r
++ struct ibv_device **dev_list;\r
++ int i = 0;\r
++\r
++ dev_list = ibv_get_device_list(NULL);\r
++ if (!dev_list) {\r
++ fprintf(stderr, "Failed to get IB devices list");\r
++ return -1;\r
++ }\r
++\r
++ if (ib_devname) {\r
++ for (; dev_list[i]; ++i) {\r
++ if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))\r
++ break;\r
++ }\r
++ }\r
++ if (!dev_list[i]) {\r
++ fprintf(stderr, "IB device %s not found\n",\r
++ ib_devname ? ib_devname : "");\r
++ return -1;\r
++ }\r
++\r
++ ctx.context = ibv_open_device(dev_list[i]);\r
++ if (!ctx.context) {\r
++ fprintf(stderr, "Couldn't get context for %s\n",\r
++ ibv_get_device_name(dev_list[i]));\r
++ return -1;\r
++ }\r
++\r
++ ibv_free_device_list(dev_list);\r
++ return 0;\r
++}\r
++\r
++static int create_qps(void)\r
++{\r
++ struct ibv_qp_init_attr init;\r
++ struct ibv_qp_attr mod;\r
++ int i;\r
++\r
++ for (i = 0; i < ctx.num_clients; ++i) {\r
++\r
++ memset(&init, 0, sizeof init);\r
++ init.qp_type = IBV_QPT_XRC_RECV;\r
++ init.ext.xrc_recv.xrcd = ctx.xrcd;\r
++\r
++ ctx.recv_qp[i] = ibv_create_qp(ctx.pd, &init);\r
++ if (!ctx.recv_qp[i]) {\r
++ fprintf(stderr, "Couldn't create recv QP[%d] errno %d\n",\r
++ i, errno);\r
++ return 1;\r
++ }\r
++\r
++ mod.qp_state = IBV_QPS_INIT;\r
++ mod.pkey_index = 0;\r
++ mod.port_num = ctx.ib_port;\r
++ mod.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;\r
++\r
++ if (ibv_modify_qp(ctx.recv_qp[i], &mod,\r
++ IBV_QP_STATE | IBV_QP_PKEY_INDEX |\r
++ IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {\r
++ fprintf(stderr, "Failed to modify recv QP[%d] to INIT\n", i);\r
++ return 1;\r
++ }\r
++\r
++ memset(&init, 0, sizeof init);\r
++ init.qp_type = IBV_QPT_XRC_SEND;\r
++ init.send_cq = ctx.send_cq;\r
++ init.cap.max_send_wr = ctx.num_clients * ctx.num_tests;\r
++ init.cap.max_send_sge = 1;\r
++\r
++ ctx.send_qp[i] = ibv_create_qp(ctx.pd, &init);\r
++ if (!ctx.send_qp[i]) {\r
++ fprintf(stderr, "Couldn't create send QP[%d] errno %d\n",\r
++ i, errno);\r
++ return 1;\r
++ }\r
++\r
++ mod.qp_state = IBV_QPS_INIT;\r
++ mod.pkey_index = 0;\r
++ mod.port_num = ctx.ib_port;\r
++ mod.qp_access_flags = 0;\r
++\r
++ if (ibv_modify_qp(ctx.send_qp[i], &mod,\r
++ IBV_QP_STATE | IBV_QP_PKEY_INDEX |\r
++ IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {\r
++ fprintf(stderr, "Failed to modify send QP[%d] to INIT\n", i);\r
++ return 1;\r
++ }\r
++ }\r
++\r
++ return 0;\r
++}\r
++\r
++static int pp_init_ctx(char *ib_devname)\r
++{\r
++ struct ibv_srq_init_attr attr;\r
++\r
++ ctx.recv_qp = calloc(ctx.num_clients, sizeof *ctx.recv_qp);\r
++ ctx.send_qp = calloc(ctx.num_clients, sizeof *ctx.send_qp);\r
++ ctx.rem_dest = calloc(ctx.num_clients, sizeof *ctx.rem_dest);\r
++ if (!ctx.recv_qp || !ctx.send_qp || !ctx.rem_dest)\r
++ return 1;\r
++\r
++ if (open_device(ib_devname)) {\r
++ fprintf(stderr, "Failed to open device\n");\r
++ return 1;\r
++ }\r
++\r
++ ctx.lid = pp_get_local_lid(ctx.context, ctx.ib_port);\r
++ if (ctx.lid < 0) {\r
++ fprintf(stderr, "Failed to get SLID\n");\r
++ return 1;\r
++ }\r
++\r
++ ctx.buf = memalign(page_size, ctx.size);\r
++ if (!ctx.buf) {\r
++ fprintf(stderr, "Couldn't allocate work buf.\n");\r
++ return 1;\r
++ }\r
++\r
++ memset(ctx.buf, 0, ctx.size);\r
++\r
++ if (ctx.use_event) {\r
++ ctx.channel = ibv_create_comp_channel(ctx.context);\r
++ if (!ctx.channel) {\r
++ fprintf(stderr, "Couldn't create completion channel\n");\r
++ return 1;\r
++ }\r
++ }\r
++\r
++ ctx.pd = ibv_alloc_pd(ctx.context);\r
++ if (!ctx.pd) {\r
++ fprintf(stderr, "Couldn't allocate PD\n");\r
++ return 1;\r
++ }\r
++\r
++ ctx.mr = ibv_reg_mr(ctx.pd, ctx.buf, ctx.size, IBV_ACCESS_LOCAL_WRITE);\r
++ if (!ctx.mr) {\r
++ fprintf(stderr, "Couldn't register MR\n");\r
++ return 1;\r
++ }\r
++\r
++ ctx.fd = open("/tmp/xrc_domain", O_CREAT);\r
++ if (ctx.fd < 0) {\r
++ fprintf(stderr,\r
++ "Couldn't create the file for the XRC Domain "\r
++ "but not stopping %d\n", errno);\r
++ ctx.fd = -1;\r
++ }\r
++\r
++ ctx.xrcd = ibv_open_xrcd(ctx.context, ctx.fd, O_CREAT);\r
++ if (!ctx.xrcd) {\r
++ fprintf(stderr, "Couldn't Open the XRC Domain %d\n", errno);\r
++ return 1;\r
++ }\r
++\r
++ ctx.recv_cq = ibv_create_cq(ctx.context, ctx.num_clients, &ctx.recv_cq,\r
++ ctx.channel, 0);\r
++ if (!ctx.recv_cq) {\r
++ fprintf(stderr, "Couldn't create recv CQ\n");\r
++ return 1;\r
++ }\r
++\r
++ if (ctx.use_event) {\r
++ if (ibv_req_notify_cq(ctx.recv_cq, 0)) {\r
++ fprintf(stderr, "Couldn't request CQ notification\n");\r
++ return 1;\r
++ }\r
++ }\r
++\r
++ ctx.send_cq = ibv_create_cq(ctx.context, ctx.num_clients, NULL, NULL, 0);\r
++ if (!ctx.send_cq) {\r
++ fprintf(stderr, "Couldn't create send CQ\n");\r
++ return 1;\r
++ }\r
++\r
++ memset(&attr, 0, sizeof attr);\r
++ attr.srq_type = IBV_SRQT_XRC;\r
++ attr.attr.max_wr = ctx.num_clients;\r
++ attr.attr.max_sge = 1;\r
++ attr.ext.xrc.xrcd = ctx.xrcd;\r
++ attr.ext.xrc.cq = ctx.recv_cq;\r
++\r
++ ctx.srq = ibv_create_xsrq(ctx.pd, &attr);\r
++ if (!ctx.srq) {\r
++ fprintf(stderr, "Couldn't create SRQ\n");\r
++ return 1;\r
++ }\r
++\r
++ if (create_qps())\r
++ return 1;\r
++\r
++ return 0;\r
++}\r
++\r
++static int send_local_dest(int sockfd, int index)\r
++{\r
++ char msg[MSG_SIZE];\r
++\r
++ ctx.rem_dest[index].recv_psn = lrand48() & 0xffffff;\r
++\r
++ printf(ADDR_FORMAT, "local", ctx.lid, ctx.recv_qp[index]->qp_num,\r
++ ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,\r
++ ctx.srq->ext.xrc.srq_num);\r
++\r
++ sprintf(msg, MSG_FORMAT, ctx.lid, ctx.recv_qp[index]->qp_num,\r
++ ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,\r
++ ctx.srq->ext.xrc.srq_num);\r
++\r
++ if (write(sockfd, msg, MSG_SIZE) != MSG_SIZE) {\r
++ fprintf(stderr, "Couldn't send local address\n");\r
++ return -1;\r
++ }\r
++\r
++ return 0;\r
++}\r
++\r
++static int recv_remote_dest(int sockfd, int index)\r
++{\r
++ struct pingpong_dest *rem_dest;\r
++ char msg[MSG_SIZE];\r
++ int n = 0, r;\r
++\r
++ while (n < MSG_SIZE) {\r
++ r = read(sockfd, msg + n, MSG_SIZE - n);\r
++ if (r < 0) {\r
++ perror("client read");\r
++ fprintf(stderr,\r
++ "%d/%d: Couldn't read remote address [%d]\n",\r
++ n, MSG_SIZE, index);\r
++ return -1;\r
++ }\r
++ n += r;\r
++ }\r
++\r
++ rem_dest = &ctx.rem_dest[index];\r
++ sscanf(msg, MSG_SSCAN, &rem_dest->lid, &rem_dest->recv_qpn,\r
++ &rem_dest->send_qpn, &rem_dest->send_psn, &rem_dest->srqn);\r
++\r
++ printf(ADDR_FORMAT, "remote", rem_dest->lid, rem_dest->recv_qpn,\r
++ rem_dest->send_qpn, rem_dest->send_psn, rem_dest->srqn);\r
++\r
++ return 0;\r
++}\r
++\r
++static int connect_qps(int index)\r
++{\r
++ struct ibv_qp_attr attr;\r
++\r
++ memset(&attr, 0, sizeof attr);\r
++ attr.qp_state = IBV_QPS_RTR;\r
++ attr.dest_qp_num = ctx.rem_dest[index].send_qpn;\r
++ attr.path_mtu = ctx.mtu;\r
++ attr.rq_psn = ctx.rem_dest[index].send_psn;\r
++ attr.min_rnr_timer = 12;\r
++ attr.ah_attr.dlid = ctx.rem_dest[index].lid;\r
++ attr.ah_attr.sl = ctx.sl;\r
++ attr.ah_attr.port_num = ctx.ib_port;\r
++\r
++ if (ibv_modify_qp(ctx.recv_qp[index], &attr,\r
++ IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |\r
++ IBV_QP_DEST_QPN | IBV_QP_RQ_PSN |\r
++ IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) {\r
++ fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index);\r
++ return 1;\r
++ }\r
++\r
++ memset(&attr, 0, sizeof attr);\r
++ attr.qp_state = IBV_QPS_RTS;\r
++ attr.timeout = 14;\r
++ attr.sq_psn = ctx.rem_dest[index].recv_psn;\r
++\r
++ if (ibv_modify_qp(ctx.recv_qp[index], &attr,\r
++ IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN)) {\r
++ fprintf(stderr, "Failed to modify recv QP[%d] to RTS\n", index);\r
++ return 1;\r
++ }\r
++\r
++ memset(&attr, 0, sizeof attr);\r
++ attr.qp_state = IBV_QPS_RTR;\r
++ attr.dest_qp_num = ctx.rem_dest[index].recv_qpn;\r
++ attr.path_mtu = ctx.mtu;\r
++ attr.rq_psn = ctx.rem_dest[index].send_psn;\r
++ attr.ah_attr.dlid = ctx.rem_dest[index].lid;\r
++ attr.ah_attr.sl = ctx.sl;\r
++ attr.ah_attr.port_num = ctx.ib_port;\r
++\r
++ if (ibv_modify_qp(ctx.send_qp[index], &attr,\r
++ IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |\r
++ IBV_QP_DEST_QPN | IBV_QP_RQ_PSN)) {\r
++ fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index);\r
++ return 1;\r
++ }\r
++\r
++ memset(&attr, 0, sizeof attr);\r
++ attr.qp_state = IBV_QPS_RTS;\r
++ attr.timeout = 14;\r
++ attr.retry_cnt = 7;\r
++ attr.rnr_retry = 7;\r
++ attr.sq_psn = ctx.rem_dest[index].recv_psn;\r
++\r
++ if (ibv_modify_qp(ctx.send_qp[index], &attr,\r
++ IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN |\r
++ IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC)) {\r
++ fprintf(stderr, "Failed to modify send QP[%d] to RTS\n", index);\r
++ return 1;\r
++ }\r
++\r
++ return 0;\r
++}\r
++\r
++static int pp_client_connect(const char *servername, int port)\r
++{\r
++ struct addrinfo *res, *t;\r
++ char *service;\r
++ int ret;\r
++ int sockfd = -1;\r
++ struct addrinfo hints = {\r
++ .ai_family = AF_UNSPEC,\r
++ .ai_socktype = SOCK_STREAM\r
++ };\r
++\r
++ if (asprintf(&service, "%d", port) < 0)\r
++ return 1;\r
++\r
++ ret = getaddrinfo(servername, service, &hints, &res);\r
++ if (ret < 0) {\r
++ fprintf(stderr, "%s for %s:%d\n", gai_strerror(ret), servername, port);\r
++ free(service);\r
++ return 1;\r
++ }\r
++\r
++ for (t = res; t; t = t->ai_next) {\r
++ sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);\r
++ if (sockfd >= 0) {\r
++ if (!connect(sockfd, t->ai_addr, t->ai_addrlen))\r
++ break;\r
++ close(sockfd);\r
++ sockfd = -1;\r
++ }\r
++ }\r
++\r
++ freeaddrinfo(res);\r
++ free(service);\r
++\r
++ if (sockfd < 0) {\r
++ fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);\r
++ return 1;\r
++ }\r
++\r
++ if (send_local_dest(sockfd, 0))\r
++ return 1;\r
++\r
++ if (recv_remote_dest(sockfd, 0))\r
++ return 1;\r
++\r
++ if (connect_qps(0))\r
++ return 1;\r
++\r
++ shutdown(sockfd, SHUT_RDWR);\r
++ close(sockfd);\r
++ return 0;\r
++}\r
++\r
++static int pp_server_connect(int port)\r
++{\r
++ struct addrinfo *res, *t;\r
++ char *service;\r
++ int ret, i, n;\r
++ int sockfd = -1, connfd;\r
++ struct addrinfo hints = {\r
++ .ai_flags = AI_PASSIVE,\r
++ .ai_family = AF_UNSPEC,\r
++ .ai_socktype = SOCK_STREAM\r
++ };\r
++\r
++ if (asprintf(&service, "%d", port) < 0)\r
++ return 1;\r
++\r
++ ret = getaddrinfo(NULL, service, &hints, &res);\r
++ if (ret < 0) {\r
++ fprintf(stderr, "%s for port %d\n", gai_strerror(ret), port);\r
++ free(service);\r
++ return 1;\r
++ }\r
++\r
++ for (t = res; t; t = t->ai_next) {\r
++ sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);\r
++ if (sockfd >= 0) {\r
++ n = 1;\r
++ setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);\r
++ if (!bind(sockfd, t->ai_addr, t->ai_addrlen))\r
++ break;\r
++ close(sockfd);\r
++ sockfd = -1;\r
++ }\r
++ }\r
++\r
++ freeaddrinfo(res);\r
++ free(service);\r
++\r
++ if (sockfd < 0) {\r
++ fprintf(stderr, "Couldn't listen to port %d\n", port);\r
++ return 1;\r
++ }\r
++\r
++ listen(sockfd, ctx.num_clients);\r
++\r
++ for (i = 0; i < ctx.num_clients; i++) {\r
++ connfd = accept(sockfd, NULL, 0);\r
++ if (connfd < 0) {\r
++ fprintf(stderr, "accept() failed for client %d\n", i);\r
++ return 1;\r
++ }\r
++\r
++ if (recv_remote_dest(connfd, i))\r
++ return 1;\r
++\r
++ if (send_local_dest(connfd, i))\r
++ return 1;\r
++\r
++ if (connect_qps(i))\r
++ return 1;\r
++\r
++ shutdown(connfd, SHUT_RDWR);\r
++ close(connfd);\r
++ }\r
++\r
++ close(sockfd);\r
++ return 0;\r
++}\r
++\r
++\r
++static int pp_close_ctx(void)\r
++{\r
++ int i;\r
++\r
++ for (i = 0; i < ctx.num_clients; ++i) {\r
++ /*\r
++ * Receive QPs may be used by other clients. Leave them, they\r
++ * will be cleaned up when the xrcd is destroyed.\r
++ */\r
++ if (ibv_destroy_qp(ctx.send_qp[i])) {\r
++ fprintf(stderr, "Couldn't destroy INI QP[%d]\n", i);\r
++ return 1;\r
++ }\r
++ }\r
++\r
++ if (ibv_destroy_srq(ctx.srq)) {\r
++ fprintf(stderr, "Couldn't destroy SRQ\n");\r
++ return 1;\r
++ }\r
++\r
++ if (!ctx.xrcd && ibv_close_xrcd(ctx.xrcd)) {\r
++ fprintf(stderr, "Couldn't close the XRC Domain\n");\r
++ return 1;\r
++ }\r
++ if (ctx.fd >= 0 && close(ctx.fd)) {\r
++ fprintf(stderr, "Couldn't close the file for the XRC Domain\n");\r
++ return 1;\r
++ }\r
++\r
++ if (ibv_destroy_cq(ctx.send_cq)) {\r
++ fprintf(stderr, "Couldn't destroy send CQ\n");\r
++ return 1;\r
++ }\r
++\r
++ if (ibv_destroy_cq(ctx.recv_cq)) {\r
++ fprintf(stderr, "Couldn't destroy recv CQ\n");\r
++ return 1;\r
++ }\r
++\r
++ if (ibv_dereg_mr(ctx.mr)) {\r
++ fprintf(stderr, "Couldn't deregister MR\n");\r
++ return 1;\r
++ }\r
++\r
++ if (ibv_dealloc_pd(ctx.pd)) {\r
++ fprintf(stderr, "Couldn't deallocate PD\n");\r
++ return 1;\r
++ }\r
++\r
++ if (ctx.channel) {\r
++ if (ibv_destroy_comp_channel(ctx.channel)) {\r
++ fprintf(stderr,\r
++ "Couldn't destroy completion channel\n");\r
++ return 1;\r
++ }\r
++ }\r
++\r
++ if (ibv_close_device(ctx.context)) {\r
++ fprintf(stderr, "Couldn't release context\n");\r
++ return 1;\r
++ }\r
++\r
++ free(ctx.buf);\r
++ free(ctx.rem_dest);\r
++ free(ctx.send_qp);\r
++ free(ctx.recv_qp);\r
++ return 0;\r
++}\r
++\r
++static int pp_post_recv(int cnt)\r
++{\r
++ struct ibv_sge sge;\r
++ struct ibv_recv_wr wr, *bad_wr;\r
++\r
++ sge.addr = (uintptr_t) ctx.buf;\r
++ sge.length = ctx.size;\r
++ sge.lkey = ctx.mr->lkey;\r
++\r
++ wr.next = NULL;\r
++ wr.wr_id = (uintptr_t) &ctx;\r
++ wr.sg_list = &sge;\r
++ wr.num_sge = 1;\r
++\r
++ while (cnt--) {\r
++ if (ibv_post_srq_recv(ctx.srq, &wr, &bad_wr)) {\r
++ fprintf(stderr, "Failed to post receive to SRQ\n");\r
++ return 1;\r
++ }\r
++ }\r
++ return 0;\r
++}\r
++\r
++/*\r
++ * Send to each client round robin on each set of xrc send/recv qp.\r
++ * Generate a completion on the last send.\r
++ */\r
++static int pp_post_send(int index)\r
++{\r
++ struct ibv_sge sge;\r
++ struct ibv_send_wr wr, *bad_wr;\r
++ int qpi;\r
++\r
++ sge.addr = (uintptr_t) ctx.buf;\r
++ sge.length = ctx.size;\r
++ sge.lkey = ctx.mr->lkey;\r
++\r
++ wr.wr_id = (uintptr_t) index;\r
++ wr.next = NULL;\r
++ wr.sg_list = &sge;\r
++ wr.num_sge = 1;\r
++ wr.opcode = IBV_WR_SEND;\r
++ wr.wr.xrc.remote_srqn = ctx.rem_dest[index].srqn;\r
++\r
++ qpi = (index + ctx.rem_dest[index].pp_cnt) % ctx.num_clients;\r
++ wr.send_flags = (++ctx.rem_dest[index].pp_cnt >= ctx.num_tests) ?\r
++ IBV_SEND_SIGNALED : 0;\r
++\r
++ return ibv_post_send(ctx.send_qp[qpi], &wr, &bad_wr);\r
++}\r
++\r
++static int find_qp(int qpn)\r
++{\r
++ int i;\r
++\r
++ if (ctx.num_clients == 1)\r
++ return 0;\r
++\r
++ for (i = 0; i < ctx.num_clients; ++i)\r
++ if (ctx.recv_qp[i]->qp_num == qpn)\r
++ return i;\r
++\r
++ fprintf(stderr, "Unable to find qp %x\n", qpn);\r
++ return 0;\r
++}\r
++\r
++static int get_cq_event(void)\r
++{\r
++ struct ibv_cq *ev_cq;\r
++ void *ev_ctx;\r
++\r
++ if (ibv_get_cq_event(ctx.channel, &ev_cq, &ev_ctx)) {\r
++ fprintf(stderr, "Failed to get cq_event\n");\r
++ return 1;\r
++ }\r
++\r
++ if (ev_cq != ctx.recv_cq) {\r
++ fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);\r
++ return 1;\r
++ }\r
++\r
++ if (ibv_req_notify_cq(ctx.recv_cq, 0)) {\r
++ fprintf(stderr, "Couldn't request CQ notification\n");\r
++ return 1;\r
++ }\r
++\r
++ return 0;\r
++}\r
++\r
++static void init(void)\r
++{\r
++ srand48(getpid() * time(NULL));\r
++\r
++ ctx.size = 4096;\r
++ ctx.ib_port = 1;\r
++ ctx.num_clients = 1;\r
++ ctx.num_tests = 5;\r
++ ctx.mtu = IBV_MTU_2048;\r
++ ctx.sl = 0;\r
++}\r
++\r
++static void usage(const char *argv0)\r
++{\r
++ printf("Usage:\n");\r
++ printf(" %s start a server and wait for connection\n", argv0);\r
++ printf(" %s <host> connect to server at <host>\n", argv0);\r
++ printf("\n");\r
++ printf("Options:\n");\r
++ printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");\r
++ printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");\r
++ printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");\r
++ printf(" -s, --size=<size> size of message to exchange (default 4096)\n");\r
++ printf(" -m, --mtu=<size> path MTU (default 2048)\n");\r
++ printf(" -c, --clients=<n> number of clients (on server only, default 1)\n");\r
++ printf(" -n, --num_tests=<n> number of tests per client (default 5)\n");\r
++ printf(" -l, --sl=<sl> service level value\n");\r
++ printf(" -e, --events sleep on CQ events (default poll)\n");\r
++}\r
++\r
++int main(int argc, char *argv[])\r
++{\r
++ char *ib_devname = NULL;\r
++ char *servername = NULL;\r
++ int port = 18515;\r
++ int i, total, cnt = 0;\r
++ int ne, qpi, num_cq_events = 0;\r
++ struct ibv_wc wc;\r
++\r
++ init();\r
++ while (1) {\r
++ int c;\r
++\r
++ static struct option long_options[] = {\r
++ { .name = "port", .has_arg = 1, .val = 'p' },\r
++ { .name = "ib-dev", .has_arg = 1, .val = 'd' },\r
++ { .name = "ib-port", .has_arg = 1, .val = 'i' },\r
++ { .name = "size", .has_arg = 1, .val = 's' },\r
++ { .name = "mtu", .has_arg = 1, .val = 'm' },\r
++ { .name = "clients", .has_arg = 1, .val = 'c' },\r
++ { .name = "num_tests", .has_arg = 1, .val = 'n' },\r
++ { .name = "sl", .has_arg = 1, .val = 'l' },\r
++ { .name = "events", .has_arg = 0, .val = 'e' },\r
++ { 0 }\r
++ };\r
++\r
++ c = getopt_long(argc, argv, "p:d:i:s:m:c:n:l:e", long_options,\r
++ NULL);\r
++ if (c == -1)\r
++ break;\r
++\r
++ switch (c) {\r
++ case 'p':\r
++ port = strtol(optarg, NULL, 0);\r
++ if (port < 0 || port > 65535) {\r
++ usage(argv[0]);\r
++ return 1;\r
++ }\r
++ break;\r
++ case 'd':\r
++ ib_devname = strdupa(optarg);\r
++ break;\r
++ case 'i':\r
++ ctx.ib_port = strtol(optarg, NULL, 0);\r
++ if (ctx.ib_port < 0) {\r
++ usage(argv[0]);\r
++ return 1;\r
++ }\r
++ break;\r
++ case 's':\r
++ ctx.size = strtol(optarg, NULL, 0);\r
++ break;\r
++ case 'm':\r
++ ctx.mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));\r
++ if (ctx.mtu < 0) {\r
++ usage(argv[0]);\r
++ return 1;\r
++ }\r
++ break;\r
++ case 'c':\r
++ ctx.num_clients = strtol(optarg, NULL, 0);\r
++ break;\r
++ case 'n':\r
++ ctx.num_tests = strtol(optarg, NULL, 0);\r
++ break;\r
++ case 'l':\r
++ ctx.sl = strtol(optarg, NULL, 0);\r
++ break;\r
++ case 'e':\r
++ ctx.use_event = 1;\r
++ break;\r
++ default:\r
++ usage(argv[0]);\r
++ return 1;\r
++ }\r
++ }\r
++\r
++ if (optind == argc - 1) {\r
++ servername = strdupa(argv[optind]);\r
++ ctx.num_clients = 1;\r
++ } else if (optind < argc) {\r
++ usage(argv[0]);\r
++ return 1;\r
++ }\r
++\r
++ page_size = sysconf(_SC_PAGESIZE);\r
++\r
++ if (pp_init_ctx(ib_devname))\r
++ return 1;\r
++\r
++ if (pp_post_recv(ctx.num_clients)) {\r
++ fprintf(stderr, "Couldn't post receives\n");\r
++ return 1;\r
++ }\r
++\r
++ if (servername) {\r
++ if (pp_client_connect(servername, port))\r
++ return 1;\r
++ } else {\r
++ if (pp_server_connect(port))\r
++ return 1;\r
++\r
++ for (i = 0; i < ctx.num_clients; i++)\r
++ pp_post_send(i);\r
++ }\r
++\r
++ total = ctx.num_clients * ctx.num_tests;\r
++ while (cnt < total) {\r
++ if (ctx.use_event) {\r
++ if (get_cq_event())\r
++ return 1;\r
++\r
++ ++num_cq_events;\r
++ }\r
++\r
++ do {\r
++ ne = ibv_poll_cq(ctx.recv_cq, 1, &wc);\r
++ if (ne < 0) {\r
++ fprintf(stderr, "Error polling cq %d\n", ne);\r
++ return 1;\r
++ } else if (ne == 0) {\r
++ break;\r
++ }\r
++\r
++ if (wc.status) {\r
++ fprintf(stderr, "Work completion error %d\n", wc.status);\r
++ return 1;\r
++ }\r
++\r
++ pp_post_recv(ne);\r
++ qpi = find_qp(wc.qp_num);\r
++ if (ctx.rem_dest[qpi].pp_cnt < ctx.num_tests)\r
++ pp_post_send(qpi);\r
++ cnt += ne;\r
++ } while (ne > 0);\r
++ }\r
++\r
++ for (cnt = 0; cnt < ctx.num_clients; cnt += ne) {\r
++ ne = ibv_poll_cq(ctx.send_cq, 1, &wc);\r
++ if (ne < 0) {\r
++ fprintf(stderr, "Error polling cq %d\n", ne);\r
++ return 1;\r
++ }\r
++ }\r
++\r
++ if (ctx.use_event)\r
++ ibv_ack_cq_events(ctx.recv_cq, num_cq_events);\r
++\r
++ if (pp_close_ctx())\r
++ return 1;\r
++\r
++ return 0;\r
++}
projects / ~shefty / libibverbs.git / commitdiff