From 28e86eeef69eecbc674301c28a006871c680319c Mon Sep 17 00:00:00 2001 From: sleybo Date: Mon, 19 Feb 2007 15:45:06 +0000 Subject: [PATCH] [TOOLS] add read_bw test and cosmetics git-svn-id: svn://openib.tc.cornell.edu/gen1@598 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/tools/perftests/user/dirs | 3 +- trunk/tools/perftests/user/read_bw/SOURCES | 28 + trunk/tools/perftests/user/read_bw/makefile | 7 + trunk/tools/perftests/user/read_bw/read_bw.c | 787 ++++++++++++++++++ trunk/tools/perftests/user/read_bw/read_bw.rc | 47 ++ .../tools/perftests/user/send_lat/send_lat.c | 41 +- .../tools/perftests/user/write_bw/write_bw.c | 11 +- 7 files changed, 878 insertions(+), 46 deletions(-) create mode 100644 trunk/tools/perftests/user/read_bw/SOURCES create mode 100644 trunk/tools/perftests/user/read_bw/makefile create mode 100644 trunk/tools/perftests/user/read_bw/read_bw.c create mode 100644 trunk/tools/perftests/user/read_bw/read_bw.rc diff --git a/trunk/tools/perftests/user/dirs b/trunk/tools/perftests/user/dirs index 3e83a603..615798e1 100644 --- a/trunk/tools/perftests/user/dirs +++ b/trunk/tools/perftests/user/dirs @@ -2,6 +2,7 @@ DIRS=\ send_bw \ send_lat \ write_lat \ - write_bw + write_bw \ + read_bw # write_bw_postlist rdma_lat rdma_bw send_lat write_lat write_bw read_lat read_bwr diff --git a/trunk/tools/perftests/user/read_bw/SOURCES b/trunk/tools/perftests/user/read_bw/SOURCES new file mode 100644 index 00000000..e08f73a6 --- /dev/null +++ b/trunk/tools/perftests/user/read_bw/SOURCES @@ -0,0 +1,28 @@ +TARGETNAME=ib_read_bw +TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE=PROGRAM +UMTYPE=console +USE_CRTDLL=1 + +C_DEFINES=$(C_DEFINES) /D__WIN__ + +SOURCES=read_bw.rc \ + ..\getopt.c \ + ..\perf_utils.c \ + read_bw.c + +INCLUDES=..;..\..\..\..\inc;..\..\..\..\inc\user + +RCOPTIONS=/I..\..\win\include + +TARGETLIBS= \ + $(DDK_LIB_PATH)\Ws2_32.lib \ +!if $(FREEBUILD) + $(TARGETPATH)\*\complib.lib \ + $(TARGETPATH)\*\ibal.lib +!else + $(TARGETPATH)\*\complibd.lib \ + $(TARGETPATH)\*\ibald.lib +!endif + +MSC_WARNING_LEVEL= /W3 diff --git a/trunk/tools/perftests/user/read_bw/makefile b/trunk/tools/perftests/user/read_bw/makefile new file mode 100644 index 00000000..a0c06273 --- /dev/null +++ b/trunk/tools/perftests/user/read_bw/makefile @@ -0,0 +1,7 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# + +!INCLUDE ..\..\..\..\inc\openib.def diff --git a/trunk/tools/perftests/user/read_bw/read_bw.c b/trunk/tools/perftests/user/read_bw/read_bw.c new file mode 100644 index 00000000..797f6352 --- /dev/null +++ b/trunk/tools/perftests/user/read_bw/read_bw.c @@ -0,0 +1,787 @@ +/* + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id: read_bw.c 1955 2007-02-19 14:46:04Z sleybo $ + */ + +#include "getopt.h" +#include "perf_defs.h" +#include "get_clock.h" + +struct user_parameters { + const char *servername; + int connection_type; + int mtu; + int all; /* run all msg size */ + int iters; + int tx_depth; + int max_out_read; +}; + +static int page_size; + +cycles_t *tposted; +cycles_t *tcompleted; + + +void +pp_cq_comp_cb( + IN const ib_cq_handle_t h_cq, + IN void *cq_context ) +{ + UNUSED_PARAM( h_cq ); + UNUSED_PARAM( cq_context); + return ; +} + +static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_parameters *user_parm) +{ + + struct pingpong_context *ctx; + ib_api_status_t ib_status = IB_SUCCESS; + size_t guid_count; + ib_net64_t *ca_guid_array; + + + ctx = malloc(sizeof *ctx); + if (!ctx){ + perror("malloc"); + return NULL; + } + + memset(ctx, 0, sizeof(struct pingpong_context)); + ctx->size = size; + ctx->tx_depth = user_parm->tx_depth; + + ctx->qp = malloc(sizeof (ib_qp_handle_t)); + if (!ctx->qp) { + perror("malloc"); + return NULL; + } + + ctx->qp_attr = malloc(sizeof (ib_qp_attr_t)); + if (!ctx->qp_attr) { + perror("malloc"); + return NULL; + } + + ctx->buf = malloc( size * 2 ); + if (!ctx->buf) { + fprintf(stderr, "Couldn't allocate work buf.\n"); + return NULL; + } + + memset(ctx->buf, 0, size * 2 ); + + + /* + * Open the AL instance + */ + ib_status = ib_open_al(&ctx->al); + if(ib_status != IB_SUCCESS) + { + fprintf(stderr,"ib_open_al failed status = %d\n", ib_status); + return NULL; + } + + /* + * Get the Local CA Guids + */ + ib_status = ib_get_ca_guids(ctx->al, NULL, &guid_count); + if(ib_status != IB_INSUFFICIENT_MEMORY) + { + fprintf(stderr,"ib_get_ca_guids1 failed status = %d\n", (uint32_t)ib_status); + return NULL; + } + + /* + * If no CA's Present then return + */ + + if(guid_count == 0) + return NULL; + + + ca_guid_array = (ib_net64_t*)malloc(sizeof(ib_net64_t) * guid_count); + + ib_status = ib_get_ca_guids(ctx->al, ca_guid_array, &guid_count); + if(ib_status != IB_SUCCESS) + { + fprintf(stderr,"ib_get_ca_guids2 failed with status = %d\n", ib_status); + return NULL; + } + + /* + * Open only the first HCA + */ + /* Open the CA */ + ib_status = ib_open_ca(ctx->al ,ca_guid_array[0] ,NULL, + NULL, //ca_context + &ctx->ca); + + if(ib_status != IB_SUCCESS) + { + fprintf(stderr,"ib_open_ca failed with status = %d\n", ib_status); + return NULL; + } + + //xxx + //printf("ib_open_ca passed i=%d\n",i); + //xxx + + + + { + /* Query the CA */ + uint32_t bsize = 0; + ib_status = ib_query_ca(ctx->ca, NULL, &bsize); + if(ib_status != IB_INSUFFICIENT_MEMORY) + { + fprintf(stderr, "Failed to query device props"); + return NULL; + } + + ctx->ca_attr = (ib_ca_attr_t *)malloc(bsize); + + ib_status = ib_query_ca(ctx->ca, ctx->ca_attr, &bsize); + if(ib_status != IB_SUCCESS) + { + printf("ib_query_ca failed with status = %d\n", ib_status); + return NULL; + } + if (user_parm->mtu == 0) {/*user did not ask for specific mtu */ + if (ctx->ca_attr->dev_id == 23108) { + user_parm->mtu = 1024; + } else { + user_parm->mtu = 2048; + } + } + } + + ib_status = ib_alloc_pd(ctx->ca , + IB_PDT_NORMAL, + ctx, //pd_context + &ctx->pd); + if (ib_status != IB_SUCCESS) { + fprintf(stderr, "Couldn't allocate PD\n"); + return NULL; + } + + + /* We dont really want IBV_ACCESS_LOCAL_WRITE, but IB spec says: + * The Consumer is not allowed to assign Remote Write or Remote Atomic to + * a Memory Region that has not been assigned Local Write. */ + + + { + ib_mr_create_t mr_create; + + mr_create.length = size * 2; + + mr_create.vaddr = ctx->buf; + mr_create.access_ctrl = IB_AC_RDMA_WRITE| IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ; + + ib_status = ib_reg_mem(ctx->pd ,&mr_create ,&ctx->lkey ,&ctx->rkey ,&ctx->mr); + if (ib_status != IB_SUCCESS) { + fprintf(stderr, "Couldn't allocate MR\n"); + return NULL; + } + } + + { + ib_cq_create_t cq_create; + + cq_create.size = user_parm->tx_depth; + cq_create.h_wait_obj = NULL; + cq_create.pfn_comp_cb = pp_cq_comp_cb; + ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq); + if (ib_status != IB_SUCCESS) { + fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status); + return NULL; + } + } + + { + + ib_qp_create_t qp_create; + ib_qp_mod_t qp_modify; + ib_qp_attr_t qp_attr; + + memset(&qp_create, 0, sizeof(ib_qp_create_t)); + qp_create.h_sq_cq = ctx->scq; + qp_create.h_rq_cq = ctx->scq; + qp_create.sq_depth = user_parm->tx_depth; + qp_create.rq_depth = user_parm->tx_depth; + qp_create.sq_sge = 1; + qp_create.rq_sge = 1; + //TODO MAX_INLINE + qp_create.qp_type= IB_QPT_RELIABLE_CONN; + qp_create.sq_signaled = FALSE; + /*attr.sq_sig_all = 0;*/ + + ib_status = ib_create_qp(ctx->pd, &qp_create,NULL,NULL,&ctx->qp[0]); + if (ib_status != IB_SUCCESS){ + fprintf(stderr, "Couldn't create QP\n"); + return NULL; + } + + + + + memset(&qp_modify, 0, sizeof(ib_qp_mod_t)); + qp_modify.req_state = IB_QPS_INIT; + qp_modify.state.init.pkey_index = 0 ; + qp_modify.state.init.primary_port = (uint8_t)port; + qp_modify.state.init.access_ctrl = IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ; + + + ib_status = ib_modify_qp(ctx->qp[0], &qp_modify); + if (ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to INIT\n"); + return NULL; + } + + memset(&qp_attr, 0, sizeof(ib_qp_attr_t)); + ib_status = ib_query_qp(ctx->qp[0],&ctx->qp_attr[0]); + if (ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to INIT\n"); + return NULL; + } + fprintf(stderr, "max inline size %d\n",ctx->qp_attr[0].sq_max_inline); + } + return ctx; +} + + + + + +static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, + struct pingpong_dest *dest, struct user_parameters *user_parm,int qpindex) +{ + + ib_api_status_t ib_status; + ib_qp_mod_t attr; + memset(&attr, 0, sizeof(ib_qp_mod_t)); + + attr.req_state = IB_QPS_RTR; + switch (user_parm->mtu) { + case 256 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_256; + break; + case 512 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_512; + break; + case 1024 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_1024; + break; + case 2048 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_2048; + break; + } + printf("Mtu : %d\n", user_parm->mtu); + attr.state.rtr.dest_qp = dest->qpn;; + attr.state.rtr.rq_psn = dest->psn; + attr.state.rtr.resp_res = (uint8_t)user_parm->max_out_read; + attr.state.rtr.rnr_nak_timeout = 12; + attr.state.rtr.primary_av.grh_valid = 0; + attr.state.rtr.primary_av.dlid = dest->lid; + attr.state.rtr.primary_av.sl = 0; + attr.state.rtr.primary_av.path_bits = 0; + attr.state.rtr.primary_av.port_num = (uint8_t)port; + attr.state.rtr.primary_av.static_rate = IB_PATH_RECORD_RATE_10_GBS; + attr.state.rtr.opts = IB_MOD_QP_LOCAL_ACK_TIMEOUT | + IB_MOD_QP_RESP_RES | + IB_MOD_QP_PRIMARY_AV; + + + ib_status = ib_modify_qp(ctx->qp[0], &attr); + if(ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to RTR\n"); + return 1; + } + + + + memset(&attr, 0, sizeof(ib_qp_mod_t)); + attr.req_state = IB_QPS_RTS; + attr.state.rts.sq_psn = my_psn; + attr.state.rts.init_depth = (uint8_t)user_parm->max_out_read; + attr.state.rts.local_ack_timeout = 14; + attr.state.rts.retry_cnt = 7; + attr.state.rts.rnr_retry_cnt = 7; + attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT | + IB_MOD_QP_RETRY_CNT | + IB_MOD_QP_INIT_DEPTH | + IB_MOD_QP_LOCAL_ACK_TIMEOUT; + + ib_status = ib_modify_qp(ctx->qp[0], &attr); + if(ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to RTS\n"); + return 1; + } + return 0; +} + + +static SOCKET pp_open_port(struct pingpong_context *ctx, const char * servername, + int ib_port, int port, struct pingpong_dest **p_my_dest, + struct pingpong_dest **p_rem_dest,struct user_parameters *user_parm) +{ + struct pingpong_dest *my_dest; + struct pingpong_dest *rem_dest; + SOCKET sockfd; + int rc; + int i; + int numofqps = 1; + + /* Create connection between client and server. + * We do it by exchanging data over a TCP socket connection. */ + + + my_dest = malloc( sizeof (struct pingpong_dest) * numofqps); + if (!my_dest){ + perror("malloc"); + return INVALID_SOCKET; + } + + rem_dest = malloc(sizeof (struct pingpong_dest) * numofqps ); + if (!rem_dest){ + perror("malloc"); + return INVALID_SOCKET; + } + + sockfd = servername ? pp_client_connect(servername, port) : + pp_server_connect(port); + + if (sockfd == INVALID_SOCKET) { + printf("pp_connect_sock(%s,%d) failed (%d)!\n", + servername, port, sockfd); + return INVALID_SOCKET; + } + + + for (i =0 ;ica_attr->p_port_attr[ib_port-1].lid; + my_dest[i].psn = rand() & 0xffffff; + if (!my_dest[i].lid) { + fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n"); + return 1; + } + my_dest[i].qpn = ctx->qp_attr[i].num; + /* TBD this should be changed inot VA and different key to each qp */ + my_dest[i].rkey = ctx->rkey; + my_dest[i].vaddr = (uintptr_t)ctx->buf + ctx->size; + + printf(" local address: LID %#04x, QPN %#06x, PSN %#06x, " + "RKey %#08x VAddr %#016Lx\n", + my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, + my_dest[i].rkey, my_dest[i].vaddr); + + rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]): + pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]); + if (rc) + return INVALID_SOCKET; + printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x, " + "RKey %#08x VAddr %#016Lx\n", + rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn, + rem_dest[i].rkey, rem_dest[i].vaddr); + + if (pp_connect_ctx(ctx, ib_port, my_dest[i].psn, &rem_dest[i], user_parm,i)) + return INVALID_SOCKET; + /* An additional handshake is required *after* moving qp to RTR. + Arbitrarily reuse exch_dest for this purpose. */ + rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]): + pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]); + if (rc) + return INVALID_SOCKET; + } + *p_rem_dest = rem_dest; + *p_my_dest = my_dest; + return sockfd; +} + +static void usage(const char *argv0) +{ + printf("Usage:\n"); + printf(" %s start a server and wait for connection\n", argv0); + printf(" %s connect to server at \n", argv0); + printf("\n"); + printf("Options:\n"); + printf(" -p, --port= listen on/connect to port (default 18515)\n"); + printf(" -d, --ib-dev= use IB device (default first device found)\n"); + printf(" -i, --ib-port= use port of IB device (default 1)\n"); + printf(" -m, --mtu= mtu size (default 1024)\n"); + printf(" -o, --outs= num of outstanding read/atom(default 4)\n"); + printf(" -s, --size= size of message to exchange (default 65536)\n"); + printf(" -a, --all Run sizes from 2 till 2^23\n"); + printf(" -t, --tx-depth= size of tx queue (default 100)\n"); + printf(" -n, --iters= number of exchanges (at least 2, default 1000)\n"); + printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n"); + printf(" -V, --version display version number\n"); + printf(" -e, --events sleep on CQ events (default poll)\n"); +} + +static void print_report(unsigned int iters, unsigned size, int duplex, + cycles_t *tposted, cycles_t *tcompleted) +{ + double cycles_to_units; + uint64_t tsize; /* Transferred size, in megabytes */ + unsigned int i, j; + int opt_posted = 0, opt_completed = 0; + cycles_t opt_delta; + cycles_t t; + + + opt_delta = tcompleted[opt_posted] - tposted[opt_completed]; + + /* Find the peak bandwidth */ + for (i = 0; i < iters; ++i) + for (j = i; j < iters; ++j) { + t = (tcompleted[j] - tposted[i]) / (j - i + 1); + if (t < opt_delta) { + opt_delta = t; + opt_posted = i; + opt_completed = j; + } + } + + cycles_to_units = get_cpu_mhz() ; + + tsize = duplex ? 2 : 1; + tsize = tsize * size; + printf("%7d %d %7.2f %7.2f\n", + size,iters,tsize * cycles_to_units / opt_delta / 0x100000, + (uint64_t)tsize * iters * cycles_to_units /(tcompleted[iters - 1] - tposted[0]) / 0x100000); +} + + + +int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, + struct pingpong_dest *rem_dest, int size) +{ + ib_api_status_t ib_status; + int scnt, ccnt ; + ib_send_wr_t *bad_wr; + + ctx->list.vaddr = (uintptr_t) ctx->buf; + ctx->list.length = size; + ctx->list.lkey = ctx->lkey; + ctx->wr.remote_ops.vaddr = rem_dest->vaddr; + ctx->wr.remote_ops.rkey = rem_dest->rkey; + ctx->wr.wr_id = PINGPONG_RDMA_WRID; + ctx->wr.ds_array = &ctx->list; + ctx->wr.num_ds = 1; + ctx->wr.wr_type = WR_RDMA_READ; + ctx->wr.send_opt = IB_SEND_OPT_SIGNALED; + ctx->wr.p_next = NULL; + + scnt = 0; + ccnt = 0; + + /* Done with setup. Start the test. */ + while (scnt < user_param->iters || ccnt < user_param->iters) { + + while (scnt < user_param->iters && (scnt - ccnt) < user_param->tx_depth ) { + + tposted[scnt] = get_cycles(); + ib_status = ib_post_send(ctx->qp[0], &ctx->wr, &bad_wr); + if (ib_status != IB_SUCCESS) + { + fprintf(stderr, "Couldn't post send: scnt %d ccnt %d\n",scnt,ccnt); + return 1; + } + ++scnt; + PERF_DEBUG("scnt = %d \n",scnt); + + } + + if (ccnt < user_param->iters) + { + ib_wc_t wc; + ib_wc_t *p_wc_done,*p_wc_free; + + p_wc_free = &wc; + p_wc_done = NULL; + p_wc_free->p_next = NULL; + + + do { + ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done); + if (ib_status == IB_SUCCESS) { + tcompleted[ccnt] = get_cycles(); + if (p_wc_done->status != IB_WCS_SUCCESS) { + fprintf(stderr, "Completion wth error at %s:\n", + user_param->servername ? "client" : "server"); + fprintf(stderr, "Failed status %d: wr_id %d syndrom 0x%x\n", + p_wc_done->status, (int) p_wc_done->wr_id, p_wc_done->vendor_specific); + return 1; + } + + /*here the id is the index to the qp num */ + ++ccnt; + PERF_DEBUG("ccnt = %d \n",ccnt); + p_wc_free = p_wc_done; + p_wc_free->p_next = NULL; + p_wc_done = NULL; + } + + + } while (ib_status == IB_SUCCESS); + + if (ib_status != IB_NOT_FOUND) { + fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status); + return 12; + } + + } + } + + return(0); +} + +int __cdecl main(int argc, char *argv[]) +{ + struct pingpong_context *ctx; + struct pingpong_dest *my_dest; + struct pingpong_dest *rem_dest; + struct user_parameters user_param; + char *ib_devname = NULL; + int port = 18515; + int ib_port = 1; + unsigned size = 65536; + SOCKET sockfd = INVALID_SOCKET; + WSADATA wsaData; + int iResult; + int i = 0; + int duplex = 0; + + /* init default values to user's parameters */ + memset(&user_param, 0, sizeof(struct user_parameters)); + user_param.mtu = 0; /* signal choose default by device */ + user_param.iters = 1000; + user_param.tx_depth = 100; + user_param.servername = NULL; + user_param.connection_type = RC; + user_param.max_out_read = 4; /* the device capability on gen2 */ + /* Parameter parsing. */ + while (1) { + int c; + + static struct option long_options[] = { + { "port", 1, NULL, 'p' }, + { "ib-dev", 1, NULL, 'd' }, + { "ib-port", 1, NULL, 'i' }, + { "mtu", 1, NULL, 'm' }, + { "outs", 1, NULL, 'o' }, + { "size", 1, NULL, 's' }, + { "iters", 1, NULL, 'n' }, + { "tx-depth", 1, NULL, 't' }, + { "all", 0, NULL, 'a' }, + { "bidirectional", 0, NULL, 'b' }, + { "version", 0, NULL, 'V' }, + { 0 } + }; + c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:ba", long_options, NULL); + if (c == -1) + break; + + switch (c) { + case 'p': + port = strtol(optarg, NULL, 0); + if (port < 0 || port > 65535) { + usage(argv[0]); + return 1; + } + break; + + case 'd': + ib_devname = _strdup(optarg); + break; + case 'm': + user_param.mtu = strtol(optarg, NULL, 0); + break; + case 'o': + user_param.max_out_read = strtol(optarg, NULL, 0); + break; + case 'a': + user_param.all = ALL; + break; + case 'V': + printf("read_bw version : %.2f\n",VERSION); + return 0; + break; + case 'i': + ib_port = strtol(optarg, NULL, 0); + if (ib_port < 0) { + usage(argv[0]); + return 1; + } + break; + + case 's': + size = strtol(optarg, NULL, 0); + if (size < 1 || size > UINT_MAX / 2) { + usage(argv[0]); + return 1; + } + break; + + case 't': + user_param.tx_depth = strtol(optarg, NULL, 0); + if (user_param.tx_depth < 1) { usage(argv[0]); return 1; } + break; + + case 'n': + user_param.iters = strtol(optarg, NULL, 0); + if (user_param.iters < 2) { + usage(argv[0]); + return 1; + } + + break; + + case 'b': + duplex = 1; + break; + + default: + usage(argv[0]); + return 1; + } + } + + if (optind == argc - 1) + user_param.servername = _strdup(argv[optind]); + else if (optind < argc) { + usage(argv[0]); + return 6; + } + printf("------------------------------------------------------------------\n"); + if (duplex == 1) { + printf(" RDMA_Read Bidirectional BW Test\n"); + } else { + printf(" RDMA_Read BW Test\n"); + } + printf("Connection type : RC\n"); + /* Done with parameter parsing. Perform setup. */ + + // Initialize Winsock + iResult = WSAStartup(MAKEWORD(2,2), &wsaData); + if (iResult != NO_ERROR) { + printf("Error at WSAStartup()\n"); + return 1; + } + + + if (user_param.all == ALL) { + /*since we run all sizes */ + size = 8388608; /*2^23 */ + } + + srand(GetCurrentProcessId() * GetTickCount()); + + //TODO: get pagesize from sysinfo + page_size = 4096; + + //TODO:get the device names + + + ctx = pp_init_ctx(size,ib_port, &user_param); + if (!ctx) + return 8; + + + sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param); + if (sockfd == INVALID_SOCKET) + return 9; + + + printf("------------------------------------------------------------------\n"); + printf(" #bytes #iterations BW peak[MB/sec] BW average[MB/sec] \n"); + /* For half duplex tests, server just waits for client to exit */ + /* use dummy my_dest struct*/ + if (!user_param.servername && !duplex) { + pp_server_exch_dest(sockfd, my_dest,rem_dest); + send(sockfd, "done", sizeof "done",0); + closesocket(sockfd); + return 0; + } + + tposted = malloc(user_param.iters * sizeof *tposted); + + if (!tposted) { + perror("malloc"); + return 1; + } + + tcompleted = malloc(user_param.iters * sizeof *tcompleted); + + if (!tcompleted) { + perror("malloc"); + return 1; + } + + + + if (user_param.all == ALL) { + for (i = 1; i < 24 ; ++i) { + size = 1 << i; + if(run_iter(ctx, &user_param, rem_dest, size)) + return 17; + print_report(user_param.iters, size, duplex, tposted, tcompleted); + } + } else { + if(run_iter(ctx, &user_param, rem_dest, size)) + return 18; + print_report(user_param.iters, size, duplex, tposted, tcompleted); + } + + if (user_param.servername) { + pp_client_exch_dest(sockfd, my_dest,rem_dest); + } else { + pp_server_exch_dest(sockfd, my_dest,rem_dest); + } + + send(sockfd, "done", sizeof "done",0); + closesocket(sockfd); + + free(tposted); + free(tcompleted); + + printf("------------------------------------------------------------------\n"); + return 0; +} diff --git a/trunk/tools/perftests/user/read_bw/read_bw.rc b/trunk/tools/perftests/user/read_bw/read_bw.rc new file mode 100644 index 00000000..13e5f43d --- /dev/null +++ b/trunk/tools/perftests/user/read_bw/read_bw.rc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2005 Mellanox Technologies. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id: write_bw.rc 1611 2006-08-20 14:48:55Z sleybo $ + */ + + +#include + +#define VER_FILETYPE VFT_APP +#define VER_FILESUBTYPE VFT2_UNKNOWN + +#ifdef _DEBUG_ +#define VER_FILEDESCRIPTION_STR "RDMA read Bandwidth Test (Debug)" +#else +#define VER_FILEDESCRIPTION_STR "RDMA read Bandwidth Test " +#endif + +#define VER_INTERNALNAME_STR "ib_read_bw.exe" +#define VER_ORIGINALFILENAME_STR "ib_read_bw.exe" + +#include diff --git a/trunk/tools/perftests/user/send_lat/send_lat.c b/trunk/tools/perftests/user/send_lat/send_lat.c index d519c1c3..79e8126a 100644 --- a/trunk/tools/perftests/user/send_lat/send_lat.c +++ b/trunk/tools/perftests/user/send_lat/send_lat.c @@ -54,7 +54,6 @@ struct user_parameters { int all; /* run all msg size */ int iters; int tx_depth; - int use_event; }; struct report_options { @@ -208,15 +207,6 @@ static struct pingpong_context *pp_init_ctx(unsigned int size,int port,struct us } } } - if (user_parm->use_event) { -//PORTED ctx->channel = ibv_create_comp_channel(ctx->context); - ctx->channel = NULL;//remove when PORTED - if (!ctx->channel) { - fprintf(stderr, "Couldn't create completion channel\n"); - return NULL; - } - } else - ctx->channel = NULL; ib_status = ib_alloc_pd(ctx->ca , IB_PDT_NORMAL, @@ -727,7 +717,7 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, do { ib_status = ib_poll_cq(ctx->rcq,&p_wc_free, &p_wc_done); - } while (!user_param->use_event && ib_status == IB_NOT_FOUND); + } while (ib_status == IB_NOT_FOUND); if (ib_status != IB_SUCCESS) { fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status); @@ -781,33 +771,11 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, p_wc_done = NULL; p_wc_free->p_next = NULL; -#if PORTED - if (user_param->use_event) { - struct ibv_cq *ev_cq; - void *ev_ctx; - - if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get send cq_event\n"); - return 1; - } - - if (ev_cq != ctx->scq) { - fprintf(stderr, "CQ event for unknown SCQ %p\n", ev_cq); - return 1; - } - - if (ibv_req_notify_cq(ctx->scq, 0)) { - fprintf(stderr, "Couldn't request SCQ notification\n"); - return 1; - } - } - -#endif /* poll on scq */ do { ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done); - } while (!user_param->use_event && ib_status == IB_NOT_FOUND); + } while (ib_status == IB_NOT_FOUND); if (ib_status != IB_SUCCESS) { fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status); @@ -866,7 +834,6 @@ int __cdecl main(int argc, char *argv[]) user_param.iters = 1000; user_param.tx_depth = 50; user_param.servername = NULL; - user_param.use_event = 0; /* Parameter parsing. */ while (1) { int c; @@ -886,7 +853,6 @@ int __cdecl main(int argc, char *argv[]) { "report-histogram", 0,NULL, 'H' }, { "report-unsorted", 0,NULL, 'U' }, { "version", 0,NULL, 'V' }, - { "events", 0,NULL, 'e' }, { 0 } }; @@ -909,9 +875,6 @@ int __cdecl main(int argc, char *argv[]) user_param.connection_type=UD; /* default is 0 for any other option RC*/ break; - case 'e': - ++user_param.use_event; - break; case 'm': user_param.mtu = strtol(optarg, NULL, 0); break; diff --git a/trunk/tools/perftests/user/write_bw/write_bw.c b/trunk/tools/perftests/user/write_bw/write_bw.c index 951975c7..05a4e714 100644 --- a/trunk/tools/perftests/user/write_bw/write_bw.c +++ b/trunk/tools/perftests/user/write_bw/write_bw.c @@ -73,7 +73,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user ib_api_status_t ib_status = IB_SUCCESS; size_t guid_count; ib_net64_t *ca_guid_array; - int counter; + int counter; ctx = malloc(sizeof *ctx); if (!ctx){ @@ -651,8 +651,8 @@ int __cdecl main(int argc, char *argv[]) unsigned size = 65536; SOCKET sockfd = INVALID_SOCKET; WSADATA wsaData; + int iResult; int i = 0; - int iResult; int duplex = 0; @@ -777,7 +777,6 @@ int __cdecl main(int argc, char *argv[]) printf(" RDMA_Write BW Test\n"); } - printf("Inline data is used up to 400 bytes message\n"); printf("Number of qp's running %d\n",user_param.numofqps); printf("Number of iterations %d\n",user_param.iters); printf("Massege size %d\n",size); @@ -787,10 +786,10 @@ int __cdecl main(int argc, char *argv[]) printf("Connection type : UC\n"); } if (user_param.maxpostsofqpiniteration > user_param.tx_depth ) { - printf("Can not post more than tx_depth , adjusting number of post to tx_depth\n"); - user_param.maxpostsofqpiniteration = user_param.tx_depth; + printf("Can not post more than tx_depth , adjusting number of post to tx_depth\n"); + user_param.maxpostsofqpiniteration = user_param.tx_depth; } else { - printf("Each Qp will post up to %d messages each time\n",user_param.maxpostsofqpiniteration); + printf("Each Qp will post up to %d messages each time\n",user_param.maxpostsofqpiniteration); } /* Done with parameter parsing. Perform setup. */ -- 2.41.0