--- /dev/null
+/*
+ * Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: read_bw.c 1955 2007-02-19 14:46:04Z sleybo $
+ */
+
+#include "getopt.h"
+#include "perf_defs.h"
+#include "get_clock.h"
+
+struct user_parameters {
+ const char *servername;
+ int connection_type;
+ int mtu;
+ int all; /* run all msg size */
+ int iters;
+ int tx_depth;
+ int max_out_read;
+};
+
+static int page_size;
+
+cycles_t *tposted;
+cycles_t *tcompleted;
+
+
+void
+pp_cq_comp_cb(
+ IN const ib_cq_handle_t h_cq,
+ IN void *cq_context )
+{
+ UNUSED_PARAM( h_cq );
+ UNUSED_PARAM( cq_context);
+ return ;
+}
+
+static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_parameters *user_parm)
+{
+
+ struct pingpong_context *ctx;
+ ib_api_status_t ib_status = IB_SUCCESS;
+ size_t guid_count;
+ ib_net64_t *ca_guid_array;
+
+
+ ctx = malloc(sizeof *ctx);
+ if (!ctx){
+ perror("malloc");
+ return NULL;
+ }
+
+ memset(ctx, 0, sizeof(struct pingpong_context));
+ ctx->size = size;
+ ctx->tx_depth = user_parm->tx_depth;
+
+ ctx->qp = malloc(sizeof (ib_qp_handle_t));
+ if (!ctx->qp) {
+ perror("malloc");
+ return NULL;
+ }
+
+ ctx->qp_attr = malloc(sizeof (ib_qp_attr_t));
+ if (!ctx->qp_attr) {
+ perror("malloc");
+ return NULL;
+ }
+
+ ctx->buf = malloc( size * 2 );
+ if (!ctx->buf) {
+ fprintf(stderr, "Couldn't allocate work buf.\n");
+ return NULL;
+ }
+
+ memset(ctx->buf, 0, size * 2 );
+
+
+ /*
+ * Open the AL instance
+ */
+ ib_status = ib_open_al(&ctx->al);
+ if(ib_status != IB_SUCCESS)
+ {
+ fprintf(stderr,"ib_open_al failed status = %d\n", ib_status);
+ return NULL;
+ }
+
+ /*
+ * Get the Local CA Guids
+ */
+ ib_status = ib_get_ca_guids(ctx->al, NULL, &guid_count);
+ if(ib_status != IB_INSUFFICIENT_MEMORY)
+ {
+ fprintf(stderr,"ib_get_ca_guids1 failed status = %d\n", (uint32_t)ib_status);
+ return NULL;
+ }
+
+ /*
+ * If no CA's Present then return
+ */
+
+ if(guid_count == 0)
+ return NULL;
+
+
+ ca_guid_array = (ib_net64_t*)malloc(sizeof(ib_net64_t) * guid_count);
+
+ ib_status = ib_get_ca_guids(ctx->al, ca_guid_array, &guid_count);
+ if(ib_status != IB_SUCCESS)
+ {
+ fprintf(stderr,"ib_get_ca_guids2 failed with status = %d\n", ib_status);
+ return NULL;
+ }
+
+ /*
+ * Open only the first HCA
+ */
+ /* Open the CA */
+ ib_status = ib_open_ca(ctx->al ,ca_guid_array[0] ,NULL,
+ NULL, //ca_context
+ &ctx->ca);
+
+ if(ib_status != IB_SUCCESS)
+ {
+ fprintf(stderr,"ib_open_ca failed with status = %d\n", ib_status);
+ return NULL;
+ }
+
+ //xxx
+ //printf("ib_open_ca passed i=%d\n",i);
+ //xxx
+
+
+
+ {
+ /* Query the CA */
+ uint32_t bsize = 0;
+ ib_status = ib_query_ca(ctx->ca, NULL, &bsize);
+ if(ib_status != IB_INSUFFICIENT_MEMORY)
+ {
+ fprintf(stderr, "Failed to query device props");
+ return NULL;
+ }
+
+ ctx->ca_attr = (ib_ca_attr_t *)malloc(bsize);
+
+ ib_status = ib_query_ca(ctx->ca, ctx->ca_attr, &bsize);
+ if(ib_status != IB_SUCCESS)
+ {
+ printf("ib_query_ca failed with status = %d\n", ib_status);
+ return NULL;
+ }
+ if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
+ if (ctx->ca_attr->dev_id == 23108) {
+ user_parm->mtu = 1024;
+ } else {
+ user_parm->mtu = 2048;
+ }
+ }
+ }
+
+ ib_status = ib_alloc_pd(ctx->ca ,
+ IB_PDT_NORMAL,
+ ctx, //pd_context
+ &ctx->pd);
+ if (ib_status != IB_SUCCESS) {
+ fprintf(stderr, "Couldn't allocate PD\n");
+ return NULL;
+ }
+
+
+ /* We dont really want IBV_ACCESS_LOCAL_WRITE, but IB spec says:
+ * The Consumer is not allowed to assign Remote Write or Remote Atomic to
+ * a Memory Region that has not been assigned Local Write. */
+
+
+ {
+ ib_mr_create_t mr_create;
+
+ mr_create.length = size * 2;
+
+ mr_create.vaddr = ctx->buf;
+ mr_create.access_ctrl = IB_AC_RDMA_WRITE| IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ;
+
+ ib_status = ib_reg_mem(ctx->pd ,&mr_create ,&ctx->lkey ,&ctx->rkey ,&ctx->mr);
+ if (ib_status != IB_SUCCESS) {
+ fprintf(stderr, "Couldn't allocate MR\n");
+ return NULL;
+ }
+ }
+
+ {
+ ib_cq_create_t cq_create;
+
+ cq_create.size = user_parm->tx_depth;
+ cq_create.h_wait_obj = NULL;
+ cq_create.pfn_comp_cb = pp_cq_comp_cb;
+ ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq);
+ if (ib_status != IB_SUCCESS) {
+ fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status);
+ return NULL;
+ }
+ }
+
+ {
+
+ ib_qp_create_t qp_create;
+ ib_qp_mod_t qp_modify;
+ ib_qp_attr_t qp_attr;
+
+ memset(&qp_create, 0, sizeof(ib_qp_create_t));
+ qp_create.h_sq_cq = ctx->scq;
+ qp_create.h_rq_cq = ctx->scq;
+ qp_create.sq_depth = user_parm->tx_depth;
+ qp_create.rq_depth = user_parm->tx_depth;
+ qp_create.sq_sge = 1;
+ qp_create.rq_sge = 1;
+ //TODO MAX_INLINE
+ qp_create.qp_type= IB_QPT_RELIABLE_CONN;
+ qp_create.sq_signaled = FALSE;
+ /*attr.sq_sig_all = 0;*/
+
+ ib_status = ib_create_qp(ctx->pd, &qp_create,NULL,NULL,&ctx->qp[0]);
+ if (ib_status != IB_SUCCESS){
+ fprintf(stderr, "Couldn't create QP\n");
+ return NULL;
+ }
+
+
+
+
+ memset(&qp_modify, 0, sizeof(ib_qp_mod_t));
+ qp_modify.req_state = IB_QPS_INIT;
+ qp_modify.state.init.pkey_index = 0 ;
+ qp_modify.state.init.primary_port = (uint8_t)port;
+ qp_modify.state.init.access_ctrl = IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ;
+
+
+ ib_status = ib_modify_qp(ctx->qp[0], &qp_modify);
+ if (ib_status != IB_SUCCESS){
+ fprintf(stderr, "Failed to modify QP to INIT\n");
+ return NULL;
+ }
+
+ memset(&qp_attr, 0, sizeof(ib_qp_attr_t));
+ ib_status = ib_query_qp(ctx->qp[0],&ctx->qp_attr[0]);
+ if (ib_status != IB_SUCCESS){
+ fprintf(stderr, "Failed to modify QP to INIT\n");
+ return NULL;
+ }
+ fprintf(stderr, "max inline size %d\n",ctx->qp_attr[0].sq_max_inline);
+ }
+ return ctx;
+}
+
+
+
+
+
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+ struct pingpong_dest *dest, struct user_parameters *user_parm,int qpindex)
+{
+
+ ib_api_status_t ib_status;
+ ib_qp_mod_t attr;
+ memset(&attr, 0, sizeof(ib_qp_mod_t));
+
+ attr.req_state = IB_QPS_RTR;
+ switch (user_parm->mtu) {
+ case 256 :
+ attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_256;
+ break;
+ case 512 :
+ attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_512;
+ break;
+ case 1024 :
+ attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_1024;
+ break;
+ case 2048 :
+ attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_2048;
+ break;
+ }
+ printf("Mtu : %d\n", user_parm->mtu);
+ attr.state.rtr.dest_qp = dest->qpn;;
+ attr.state.rtr.rq_psn = dest->psn;
+ attr.state.rtr.resp_res = (uint8_t)user_parm->max_out_read;
+ attr.state.rtr.rnr_nak_timeout = 12;
+ attr.state.rtr.primary_av.grh_valid = 0;
+ attr.state.rtr.primary_av.dlid = dest->lid;
+ attr.state.rtr.primary_av.sl = 0;
+ attr.state.rtr.primary_av.path_bits = 0;
+ attr.state.rtr.primary_av.port_num = (uint8_t)port;
+ attr.state.rtr.primary_av.static_rate = IB_PATH_RECORD_RATE_10_GBS;
+ attr.state.rtr.opts = IB_MOD_QP_LOCAL_ACK_TIMEOUT |
+ IB_MOD_QP_RESP_RES |
+ IB_MOD_QP_PRIMARY_AV;
+
+
+ ib_status = ib_modify_qp(ctx->qp[0], &attr);
+ if(ib_status != IB_SUCCESS){
+ fprintf(stderr, "Failed to modify QP to RTR\n");
+ return 1;
+ }
+
+
+
+ memset(&attr, 0, sizeof(ib_qp_mod_t));
+ attr.req_state = IB_QPS_RTS;
+ attr.state.rts.sq_psn = my_psn;
+ attr.state.rts.init_depth = (uint8_t)user_parm->max_out_read;
+ attr.state.rts.local_ack_timeout = 14;
+ attr.state.rts.retry_cnt = 7;
+ attr.state.rts.rnr_retry_cnt = 7;
+ attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT |
+ IB_MOD_QP_RETRY_CNT |
+ IB_MOD_QP_INIT_DEPTH |
+ IB_MOD_QP_LOCAL_ACK_TIMEOUT;
+
+ ib_status = ib_modify_qp(ctx->qp[0], &attr);
+ if(ib_status != IB_SUCCESS){
+ fprintf(stderr, "Failed to modify QP to RTS\n");
+ return 1;
+ }
+ return 0;
+}
+
+
+static SOCKET pp_open_port(struct pingpong_context *ctx, const char * servername,
+ int ib_port, int port, struct pingpong_dest **p_my_dest,
+ struct pingpong_dest **p_rem_dest,struct user_parameters *user_parm)
+{
+ struct pingpong_dest *my_dest;
+ struct pingpong_dest *rem_dest;
+ SOCKET sockfd;
+ int rc;
+ int i;
+ int numofqps = 1;
+
+ /* Create connection between client and server.
+ * We do it by exchanging data over a TCP socket connection. */
+
+
+ my_dest = malloc( sizeof (struct pingpong_dest) * numofqps);
+ if (!my_dest){
+ perror("malloc");
+ return INVALID_SOCKET;
+ }
+
+ rem_dest = malloc(sizeof (struct pingpong_dest) * numofqps );
+ if (!rem_dest){
+ perror("malloc");
+ return INVALID_SOCKET;
+ }
+
+ sockfd = servername ? pp_client_connect(servername, port) :
+ pp_server_connect(port);
+
+ if (sockfd == INVALID_SOCKET) {
+ printf("pp_connect_sock(%s,%d) failed (%d)!\n",
+ servername, port, sockfd);
+ return INVALID_SOCKET;
+ }
+
+
+ for (i =0 ;i<numofqps;i ++)
+ {
+ /* Create connection between client and server.
+ * We do it by exchanging data over a TCP socket connection. */
+
+ my_dest[i].lid = ctx->ca_attr->p_port_attr[ib_port-1].lid;
+ my_dest[i].psn = rand() & 0xffffff;
+ if (!my_dest[i].lid) {
+ fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
+ return 1;
+ }
+ my_dest[i].qpn = ctx->qp_attr[i].num;
+ /* TBD this should be changed inot VA and different key to each qp */
+ my_dest[i].rkey = ctx->rkey;
+ my_dest[i].vaddr = (uintptr_t)ctx->buf + ctx->size;
+
+ printf(" local address: LID %#04x, QPN %#06x, PSN %#06x, "
+ "RKey %#08x VAddr %#016Lx\n",
+ my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn,
+ my_dest[i].rkey, my_dest[i].vaddr);
+
+ rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]):
+ pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]);
+ if (rc)
+ return INVALID_SOCKET;
+ printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x, "
+ "RKey %#08x VAddr %#016Lx\n",
+ rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn,
+ rem_dest[i].rkey, rem_dest[i].vaddr);
+
+ if (pp_connect_ctx(ctx, ib_port, my_dest[i].psn, &rem_dest[i], user_parm,i))
+ return INVALID_SOCKET;
+ /* An additional handshake is required *after* moving qp to RTR.
+ Arbitrarily reuse exch_dest for this purpose. */
+ rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]):
+ pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]);
+ if (rc)
+ return INVALID_SOCKET;
+ }
+ *p_rem_dest = rem_dest;
+ *p_my_dest = my_dest;
+ return sockfd;
+}
+
+static void usage(const char *argv0)
+{
+ printf("Usage:\n");
+ printf(" %s start a server and wait for connection\n", argv0);
+ printf(" %s <host> connect to server at <host>\n", argv0);
+ printf("\n");
+ printf("Options:\n");
+ printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
+ printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
+ printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
+ printf(" -m, --mtu=<mtu> mtu size (default 1024)\n");
+ printf(" -o, --outs=<num> num of outstanding read/atom(default 4)\n");
+ printf(" -s, --size=<size> size of message to exchange (default 65536)\n");
+ printf(" -a, --all Run sizes from 2 till 2^23\n");
+ printf(" -t, --tx-depth=<dep> size of tx queue (default 100)\n");
+ printf(" -n, --iters=<iters> number of exchanges (at least 2, default 1000)\n");
+ printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n");
+ printf(" -V, --version display version number\n");
+ printf(" -e, --events sleep on CQ events (default poll)\n");
+}
+
+static void print_report(unsigned int iters, unsigned size, int duplex,
+ cycles_t *tposted, cycles_t *tcompleted)
+{
+ double cycles_to_units;
+ uint64_t tsize; /* Transferred size, in megabytes */
+ unsigned int i, j;
+ int opt_posted = 0, opt_completed = 0;
+ cycles_t opt_delta;
+ cycles_t t;
+
+
+ opt_delta = tcompleted[opt_posted] - tposted[opt_completed];
+
+ /* Find the peak bandwidth */
+ for (i = 0; i < iters; ++i)
+ for (j = i; j < iters; ++j) {
+ t = (tcompleted[j] - tposted[i]) / (j - i + 1);
+ if (t < opt_delta) {
+ opt_delta = t;
+ opt_posted = i;
+ opt_completed = j;
+ }
+ }
+
+ cycles_to_units = get_cpu_mhz() ;
+
+ tsize = duplex ? 2 : 1;
+ tsize = tsize * size;
+ printf("%7d %d %7.2f %7.2f\n",
+ size,iters,tsize * cycles_to_units / opt_delta / 0x100000,
+ (uint64_t)tsize * iters * cycles_to_units /(tcompleted[iters - 1] - tposted[0]) / 0x100000);
+}
+
+
+
+int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
+ struct pingpong_dest *rem_dest, int size)
+{
+ ib_api_status_t ib_status;
+ int scnt, ccnt ;
+ ib_send_wr_t *bad_wr;
+
+ ctx->list.vaddr = (uintptr_t) ctx->buf;
+ ctx->list.length = size;
+ ctx->list.lkey = ctx->lkey;
+ ctx->wr.remote_ops.vaddr = rem_dest->vaddr;
+ ctx->wr.remote_ops.rkey = rem_dest->rkey;
+ ctx->wr.wr_id = PINGPONG_RDMA_WRID;
+ ctx->wr.ds_array = &ctx->list;
+ ctx->wr.num_ds = 1;
+ ctx->wr.wr_type = WR_RDMA_READ;
+ ctx->wr.send_opt = IB_SEND_OPT_SIGNALED;
+ ctx->wr.p_next = NULL;
+
+ scnt = 0;
+ ccnt = 0;
+
+ /* Done with setup. Start the test. */
+ while (scnt < user_param->iters || ccnt < user_param->iters) {
+
+ while (scnt < user_param->iters && (scnt - ccnt) < user_param->tx_depth ) {
+
+ tposted[scnt] = get_cycles();
+ ib_status = ib_post_send(ctx->qp[0], &ctx->wr, &bad_wr);
+ if (ib_status != IB_SUCCESS)
+ {
+ fprintf(stderr, "Couldn't post send: scnt %d ccnt %d\n",scnt,ccnt);
+ return 1;
+ }
+ ++scnt;
+ PERF_DEBUG("scnt = %d \n",scnt);
+
+ }
+
+ if (ccnt < user_param->iters)
+ {
+ ib_wc_t wc;
+ ib_wc_t *p_wc_done,*p_wc_free;
+
+ p_wc_free = &wc;
+ p_wc_done = NULL;
+ p_wc_free->p_next = NULL;
+
+
+ do {
+ ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
+ if (ib_status == IB_SUCCESS) {
+ tcompleted[ccnt] = get_cycles();
+ if (p_wc_done->status != IB_WCS_SUCCESS) {
+ fprintf(stderr, "Completion wth error at %s:\n",
+ user_param->servername ? "client" : "server");
+ fprintf(stderr, "Failed status %d: wr_id %d syndrom 0x%x\n",
+ p_wc_done->status, (int) p_wc_done->wr_id, p_wc_done->vendor_specific);
+ return 1;
+ }
+
+ /*here the id is the index to the qp num */
+ ++ccnt;
+ PERF_DEBUG("ccnt = %d \n",ccnt);
+ p_wc_free = p_wc_done;
+ p_wc_free->p_next = NULL;
+ p_wc_done = NULL;
+ }
+
+
+ } while (ib_status == IB_SUCCESS);
+
+ if (ib_status != IB_NOT_FOUND) {
+ fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status);
+ return 12;
+ }
+
+ }
+ }
+
+ return(0);
+}
+
+int __cdecl main(int argc, char *argv[])
+{
+ struct pingpong_context *ctx;
+ struct pingpong_dest *my_dest;
+ struct pingpong_dest *rem_dest;
+ struct user_parameters user_param;
+ char *ib_devname = NULL;
+ int port = 18515;
+ int ib_port = 1;
+ unsigned size = 65536;
+ SOCKET sockfd = INVALID_SOCKET;
+ WSADATA wsaData;
+ int iResult;
+ int i = 0;
+ int duplex = 0;
+
+ /* init default values to user's parameters */
+ memset(&user_param, 0, sizeof(struct user_parameters));
+ user_param.mtu = 0; /* signal choose default by device */
+ user_param.iters = 1000;
+ user_param.tx_depth = 100;
+ user_param.servername = NULL;
+ user_param.connection_type = RC;
+ user_param.max_out_read = 4; /* the device capability on gen2 */
+ /* Parameter parsing. */
+ while (1) {
+ int c;
+
+ static struct option long_options[] = {
+ { "port", 1, NULL, 'p' },
+ { "ib-dev", 1, NULL, 'd' },
+ { "ib-port", 1, NULL, 'i' },
+ { "mtu", 1, NULL, 'm' },
+ { "outs", 1, NULL, 'o' },
+ { "size", 1, NULL, 's' },
+ { "iters", 1, NULL, 'n' },
+ { "tx-depth", 1, NULL, 't' },
+ { "all", 0, NULL, 'a' },
+ { "bidirectional", 0, NULL, 'b' },
+ { "version", 0, NULL, 'V' },
+ { 0 }
+ };
+ c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:ba", long_options, NULL);
+ if (c == -1)
+ break;
+
+ switch (c) {
+ case 'p':
+ port = strtol(optarg, NULL, 0);
+ if (port < 0 || port > 65535) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
+ case 'd':
+ ib_devname = _strdup(optarg);
+ break;
+ case 'm':
+ user_param.mtu = strtol(optarg, NULL, 0);
+ break;
+ case 'o':
+ user_param.max_out_read = strtol(optarg, NULL, 0);
+ break;
+ case 'a':
+ user_param.all = ALL;
+ break;
+ case 'V':
+ printf("read_bw version : %.2f\n",VERSION);
+ return 0;
+ break;
+ case 'i':
+ ib_port = strtol(optarg, NULL, 0);
+ if (ib_port < 0) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
+ case 's':
+ size = strtol(optarg, NULL, 0);
+ if (size < 1 || size > UINT_MAX / 2) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
+ case 't':
+ user_param.tx_depth = strtol(optarg, NULL, 0);
+ if (user_param.tx_depth < 1) { usage(argv[0]); return 1; }
+ break;
+
+ case 'n':
+ user_param.iters = strtol(optarg, NULL, 0);
+ if (user_param.iters < 2) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ break;
+
+ case 'b':
+ duplex = 1;
+ break;
+
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (optind == argc - 1)
+ user_param.servername = _strdup(argv[optind]);
+ else if (optind < argc) {
+ usage(argv[0]);
+ return 6;
+ }
+ printf("------------------------------------------------------------------\n");
+ if (duplex == 1) {
+ printf(" RDMA_Read Bidirectional BW Test\n");
+ } else {
+ printf(" RDMA_Read BW Test\n");
+ }
+ printf("Connection type : RC\n");
+ /* Done with parameter parsing. Perform setup. */
+
+ // Initialize Winsock
+ iResult = WSAStartup(MAKEWORD(2,2), &wsaData);
+ if (iResult != NO_ERROR) {
+ printf("Error at WSAStartup()\n");
+ return 1;
+ }
+
+
+ if (user_param.all == ALL) {
+ /*since we run all sizes */
+ size = 8388608; /*2^23 */
+ }
+
+ srand(GetCurrentProcessId() * GetTickCount());
+
+ //TODO: get pagesize from sysinfo
+ page_size = 4096;
+
+ //TODO:get the device names
+
+
+ ctx = pp_init_ctx(size,ib_port, &user_param);
+ if (!ctx)
+ return 8;
+
+
+ sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param);
+ if (sockfd == INVALID_SOCKET)
+ return 9;
+
+
+ printf("------------------------------------------------------------------\n");
+ printf(" #bytes #iterations BW peak[MB/sec] BW average[MB/sec] \n");
+ /* For half duplex tests, server just waits for client to exit */
+ /* use dummy my_dest struct*/
+ if (!user_param.servername && !duplex) {
+ pp_server_exch_dest(sockfd, my_dest,rem_dest);
+ send(sockfd, "done", sizeof "done",0);
+ closesocket(sockfd);
+ return 0;
+ }
+
+ tposted = malloc(user_param.iters * sizeof *tposted);
+
+ if (!tposted) {
+ perror("malloc");
+ return 1;
+ }
+
+ tcompleted = malloc(user_param.iters * sizeof *tcompleted);
+
+ if (!tcompleted) {
+ perror("malloc");
+ return 1;
+ }
+
+
+
+ if (user_param.all == ALL) {
+ for (i = 1; i < 24 ; ++i) {
+ size = 1 << i;
+ if(run_iter(ctx, &user_param, rem_dest, size))
+ return 17;
+ print_report(user_param.iters, size, duplex, tposted, tcompleted);
+ }
+ } else {
+ if(run_iter(ctx, &user_param, rem_dest, size))
+ return 18;
+ print_report(user_param.iters, size, duplex, tposted, tcompleted);
+ }
+
+ if (user_param.servername) {
+ pp_client_exch_dest(sockfd, my_dest,rem_dest);
+ } else {
+ pp_server_exch_dest(sockfd, my_dest,rem_dest);
+ }
+
+ send(sockfd, "done", sizeof "done",0);
+ closesocket(sockfd);
+
+ free(tposted);
+ free(tcompleted);
+
+ printf("------------------------------------------------------------------\n");
+ return 0;
+}
projects / ~shefty / rdma-win.git / commitdiff