From 84f6947812e19f4ecce6ffce5f24d9567a54bbd7 Mon Sep 17 00:00:00 2001 From: sleybo Date: Mon, 19 Feb 2007 15:46:31 +0000 Subject: [PATCH] [TOOLS] add read_lat test and many cosmetics issues git-svn-id: svn://openib.tc.cornell.edu/gen1@599 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/tools/perftests/user/dirs | 1 + trunk/tools/perftests/user/get_clock.h | 15 + trunk/tools/perftests/user/perf_defs.h | 34 +- trunk/tools/perftests/user/read_lat/SOURCES | 28 + trunk/tools/perftests/user/read_lat/makefile | 7 + .../tools/perftests/user/read_lat/read_lat.c | 810 ++++++++++++++++++ .../tools/perftests/user/read_lat/read_lat.rc | 47 + .../tools/perftests/user/send_lat/send_lat.c | 23 - .../tools/perftests/user/write_bw/write_bw.c | 29 +- .../perftests/user/write_lat/write_lat.c | 107 +-- 10 files changed, 985 insertions(+), 116 deletions(-) create mode 100644 trunk/tools/perftests/user/read_lat/SOURCES create mode 100644 trunk/tools/perftests/user/read_lat/makefile create mode 100644 trunk/tools/perftests/user/read_lat/read_lat.c create mode 100644 trunk/tools/perftests/user/read_lat/read_lat.rc diff --git a/trunk/tools/perftests/user/dirs b/trunk/tools/perftests/user/dirs index 615798e1..efc41e71 100644 --- a/trunk/tools/perftests/user/dirs +++ b/trunk/tools/perftests/user/dirs @@ -3,6 +3,7 @@ DIRS=\ send_lat \ write_lat \ write_bw \ + read_lat \ read_bw # write_bw_postlist rdma_lat rdma_bw send_lat write_lat write_bw read_lat read_bwr diff --git a/trunk/tools/perftests/user/get_clock.h b/trunk/tools/perftests/user/get_clock.h index ee61458d..d46cb2c2 100644 --- a/trunk/tools/perftests/user/get_clock.h +++ b/trunk/tools/perftests/user/get_clock.h @@ -50,6 +50,21 @@ static int __cdecl cycles_compare(const void * aptr, const void * bptr) return 0; } +/* + * When there is an + * odd number of samples, the median is the middle number. + * even number of samples, the median is the mean of the + * two middle numbers. + * + */ +inline cycles_t get_median(int n, cycles_t delta[]) +{ + if (n % 2) + return(delta[n / 2] + delta[n / 2 - 1]) / 2; + else + return delta[n / 2]; +} + static inline cycles_t get_cycles() { diff --git a/trunk/tools/perftests/user/perf_defs.h b/trunk/tools/perftests/user/perf_defs.h index f6ce49ce..0719b12b 100644 --- a/trunk/tools/perftests/user/perf_defs.h +++ b/trunk/tools/perftests/user/perf_defs.h @@ -69,6 +69,10 @@ #define PINGPONG_RDMA_WRID 3 +#define SIGNAL 1 +#define MAX_INLINE 400 + + #if 0 #define PERF_ENTER printf("%s: ===>\n",__FUNCTION__); #define PERF_EXIT printf("%s: <===\n",__FUNCTION__); @@ -121,17 +125,31 @@ struct pingpong_dest { }; -static int pp_write_keys(SOCKET sockfd, const struct pingpong_dest *my_dest); -static int pp_read_keys(SOCKET sockfd, struct pingpong_dest *rem_dest); - SOCKET pp_client_connect(const char *servername, int port); +struct report_options { + int unsorted; + int histogram; + int cycles; /* report delta's in cycles, not microsec's */ +}; - int pp_client_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest, - struct pingpong_dest *rem_dest); -SOCKET pp_server_connect(int port); -int pp_server_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest, - struct pingpong_dest* rem_dest); +static int +pp_write_keys(SOCKET sockfd, const struct pingpong_dest *my_dest); +static int +pp_read_keys(SOCKET sockfd, struct pingpong_dest *rem_dest); + SOCKET + pp_client_connect(const char *servername, int port); + +int +pp_client_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest, + struct pingpong_dest *rem_dest); + +SOCKET +pp_server_connect(int port); + +int +pp_server_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest, + struct pingpong_dest* rem_dest); #endif diff --git a/trunk/tools/perftests/user/read_lat/SOURCES b/trunk/tools/perftests/user/read_lat/SOURCES new file mode 100644 index 00000000..25df9e4e --- /dev/null +++ b/trunk/tools/perftests/user/read_lat/SOURCES @@ -0,0 +1,28 @@ +TARGETNAME=ib_read_lat +TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE=PROGRAM +UMTYPE=console +USE_CRTDLL=1 + +C_DEFINES=$(C_DEFINES) /D__WIN__ + +SOURCES=read_lat.rc \ + ..\getopt.c \ + ..\perf_utils.c \ + read_lat.c + +INCLUDES=..;..\..\..\..\inc;..\..\..\..\inc\user + +RCOPTIONS=/I..\..\win\include + +TARGETLIBS= \ + $(DDK_LIB_PATH)\Ws2_32.lib \ +!if $(FREEBUILD) + $(TARGETPATH)\*\complib.lib \ + $(TARGETPATH)\*\ibal.lib +!else + $(TARGETPATH)\*\complibd.lib \ + $(TARGETPATH)\*\ibald.lib +!endif + +MSC_WARNING_LEVEL= /W3 diff --git a/trunk/tools/perftests/user/read_lat/makefile b/trunk/tools/perftests/user/read_lat/makefile new file mode 100644 index 00000000..a0c06273 --- /dev/null +++ b/trunk/tools/perftests/user/read_lat/makefile @@ -0,0 +1,7 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# + +!INCLUDE ..\..\..\..\inc\openib.def diff --git a/trunk/tools/perftests/user/read_lat/read_lat.c b/trunk/tools/perftests/user/read_lat/read_lat.c new file mode 100644 index 00000000..3cb1cd8f --- /dev/null +++ b/trunk/tools/perftests/user/read_lat/read_lat.c @@ -0,0 +1,810 @@ +/* + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. + * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler) + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id: read_lat.c 1955 2007-02-19 14:46:04Z sleybo $ + */ + +#include "getopt.h" +#include "perf_defs.h" +#include "get_clock.h" + + + + +struct user_parameters { + const char *servername; + int connection_type; + int mtu; + int all; /* run all msg size */ + int iters; + int tx_depth; + int max_out_read; +}; + +static int page_size; + +cycles_t *tstamp; + + + +void +pp_cq_comp_cb( + IN const ib_cq_handle_t h_cq, + IN void *cq_context ) +{ + UNUSED_PARAM( h_cq ); + UNUSED_PARAM( cq_context); + return ; +} + +static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_parameters *user_parm) +{ + + + struct pingpong_context *ctx; + ib_api_status_t ib_status = IB_SUCCESS; + size_t guid_count; + ib_net64_t *ca_guid_array; + + + ctx = malloc(sizeof *ctx); + if (!ctx){ + perror("malloc"); + return NULL; + } + memset(ctx, 0, sizeof(struct pingpong_context)); + ctx->size = size; + ctx->tx_depth = user_parm->tx_depth; + + ctx->qp = malloc(sizeof (ib_qp_handle_t)); + if (!ctx->qp) { + perror("malloc"); + return NULL; + } + + ctx->qp_attr = malloc(sizeof (ib_qp_attr_t)); + if (!ctx->qp_attr) { + perror("malloc"); + return NULL; + } + + ctx->buf = malloc( size * 2); + if (!ctx->buf) { + fprintf(stderr, "Couldn't allocate work buf.\n"); + return NULL; + } + + memset(ctx->buf, 0, size * 2); + + + /* + * Open the AL instance + */ + ib_status = ib_open_al(&ctx->al); + if(ib_status != IB_SUCCESS) + { + fprintf(stderr,"ib_open_al failed status = %d\n", ib_status); + return NULL; + } + + /* + * Get the Local CA Guids + */ + ib_status = ib_get_ca_guids(ctx->al, NULL, &guid_count); + if(ib_status != IB_INSUFFICIENT_MEMORY) + { + fprintf(stderr,"ib_get_ca_guids1 failed status = %d\n", (uint32_t)ib_status); + return NULL; + } + + /* + * If no CA's Present then return + */ + + if(guid_count == 0) + return NULL; + + + ca_guid_array = (ib_net64_t*)malloc(sizeof(ib_net64_t) * guid_count); + + ib_status = ib_get_ca_guids(ctx->al, ca_guid_array, &guid_count); + if(ib_status != IB_SUCCESS) + { + fprintf(stderr,"ib_get_ca_guids2 failed with status = %d\n", ib_status); + return NULL; + } + + /* + * Open only the first HCA + */ + /* Open the CA */ + ib_status = ib_open_ca(ctx->al ,ca_guid_array[0] ,NULL, + NULL, //ca_context + &ctx->ca); + + if(ib_status != IB_SUCCESS) + { + fprintf(stderr,"ib_open_ca failed with status = %d\n", ib_status); + return NULL; + } + + //xxx + //printf("ib_open_ca passed i=%d\n",i); + //xxx + + + + + { + /* Query the CA */ + uint32_t bsize = 0; + ib_status = ib_query_ca(ctx->ca, NULL, &bsize); + if(ib_status != IB_INSUFFICIENT_MEMORY) + { + fprintf(stderr, "Failed to query device props"); + return NULL; + } + + ctx->ca_attr = (ib_ca_attr_t *)malloc(bsize); + + ib_status = ib_query_ca(ctx->ca, ctx->ca_attr, &bsize); + if(ib_status != IB_SUCCESS) + { + printf("ib_query_ca failed with status = %d\n", ib_status); + return NULL; + } + if (user_parm->mtu == 0) {/*user did not ask for specific mtu */ + if (ctx->ca_attr->dev_id == 23108) { + user_parm->mtu = 1024; + } else { + user_parm->mtu = 2048; + } + } + } + + + + ib_status = ib_alloc_pd(ctx->ca , + IB_PDT_NORMAL, + ctx, //pd_context + &ctx->pd); + if (ib_status != IB_SUCCESS) { + fprintf(stderr, "Couldn't allocate PD\n"); + return NULL; + } + + + /* We dont really want IBV_ACCESS_LOCAL_WRITE, but IB spec says: + * The Consumer is not allowed to assign Remote Write or Remote Atomic to + * a Memory Region that has not been assigned Local Write. */ + + + { + ib_mr_create_t mr_create; + + mr_create.length = size * 2; + + mr_create.vaddr = ctx->buf; + mr_create.access_ctrl = IB_AC_RDMA_WRITE| IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ; + + ib_status = ib_reg_mem(ctx->pd ,&mr_create ,&ctx->lkey ,&ctx->rkey ,&ctx->mr); + if (ib_status != IB_SUCCESS) { + fprintf(stderr, "Couldn't allocate MR\n"); + return NULL; + } + } + + { + ib_cq_create_t cq_create; + + cq_create.size = user_parm->tx_depth; + cq_create.h_wait_obj = NULL; + cq_create.pfn_comp_cb = pp_cq_comp_cb; + ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq); + if (ib_status != IB_SUCCESS) { + fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status); + return NULL; + } + } + + { + + ib_qp_create_t qp_create; + ib_qp_mod_t qp_modify; + ib_qp_attr_t qp_attr; + + memset(&qp_create, 0, sizeof(ib_qp_create_t)); + qp_create.h_sq_cq = ctx->scq; + qp_create.h_rq_cq = ctx->scq; + qp_create.sq_depth = user_parm->tx_depth; + qp_create.rq_depth = 1; + qp_create.sq_sge = 1; + qp_create.rq_sge = 1; + //TODO MAX_INLINE + qp_create.qp_type= IB_QPT_RELIABLE_CONN; + qp_create.sq_signaled = FALSE; + /*attr.sq_sig_all = 0;*/ + + ib_status = ib_create_qp(ctx->pd, &qp_create,NULL,NULL,&ctx->qp[0]); + if (ib_status != IB_SUCCESS){ + fprintf(stderr, "Couldn't create QP\n"); + return NULL; + } + + + + + memset(&qp_modify, 0, sizeof(ib_qp_mod_t)); + qp_modify.req_state = IB_QPS_INIT; + qp_modify.state.init.pkey_index = 0 ; + qp_modify.state.init.primary_port = (uint8_t)port; + qp_modify.state.init.access_ctrl = IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ; + + + ib_status = ib_modify_qp(ctx->qp[0], &qp_modify); + if (ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to INIT\n"); + return NULL; + } + + + memset(&qp_attr, 0, sizeof(ib_qp_attr_t)); + ib_status = ib_query_qp(ctx->qp[0], &ctx->qp_attr[0]); + if (ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to INIT\n"); + return NULL; + } + fprintf(stderr, "max inline size %d\n",ctx->qp_attr[0].sq_max_inline); + } + return ctx; +} + + + + + +static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, + struct pingpong_dest *dest, struct user_parameters *user_parm,int qpindex) +{ + + ib_api_status_t ib_status; + ib_qp_mod_t attr; + memset(&attr, 0, sizeof(ib_qp_mod_t)); + + attr.req_state = IB_QPS_RTR; + switch (user_parm->mtu) { + case 256 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_256; + break; + case 512 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_512; + break; + case 1024 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_1024; + break; + case 2048 : + attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_2048; + break; + } + printf("Mtu : %d\n", user_parm->mtu); + attr.state.rtr.dest_qp = dest->qpn;; + attr.state.rtr.rq_psn = dest->psn; + attr.state.rtr.resp_res = (uint8_t)user_parm->max_out_read; + attr.state.rtr.rnr_nak_timeout = 12; + attr.state.rtr.primary_av.grh_valid = 0; + attr.state.rtr.primary_av.dlid = dest->lid; + attr.state.rtr.primary_av.sl = 0; + attr.state.rtr.primary_av.path_bits = 0; + attr.state.rtr.primary_av.port_num = (uint8_t)port; + attr.state.rtr.primary_av.static_rate = IB_PATH_RECORD_RATE_10_GBS; + attr.state.rtr.opts = IB_MOD_QP_LOCAL_ACK_TIMEOUT | + IB_MOD_QP_RESP_RES | + IB_MOD_QP_PRIMARY_AV; + + + ib_status = ib_modify_qp(ctx->qp[0], &attr); + if(ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to RTR\n"); + return 1; + } + + + + memset(&attr, 0, sizeof(ib_qp_mod_t)); + attr.req_state = IB_QPS_RTS; + attr.state.rts.sq_psn = my_psn; + + attr.state.rts.init_depth = (uint8_t)user_parm->max_out_read; + attr.state.rts.local_ack_timeout = 14; + attr.state.rts.retry_cnt = 7; + attr.state.rts.rnr_retry_cnt = 7; + attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT | + IB_MOD_QP_RETRY_CNT | + IB_MOD_QP_INIT_DEPTH | + IB_MOD_QP_LOCAL_ACK_TIMEOUT; + + ib_status = ib_modify_qp(ctx->qp[0], &attr); + if(ib_status != IB_SUCCESS){ + fprintf(stderr, "Failed to modify QP to RTS\n"); + return 1; + } + + return 0; +} + + +static SOCKET pp_open_port(struct pingpong_context *ctx, const char * servername, + int ib_port, int port, struct pingpong_dest **p_my_dest, + struct pingpong_dest **p_rem_dest,struct user_parameters *user_parm) +{ + //char addr_fmt[] = "%8s address: LID %#04x QPN %#06x PSN %#06x RKey %#08x VAddr %#016Lx\n"; + struct pingpong_dest *my_dest; + struct pingpong_dest *rem_dest; + SOCKET sockfd; + int rc; + int i; + int numofqps = 1; + + /* Create connection between client and server. + * We do it by exchanging data over a TCP socket connection. */ + + + my_dest = malloc( sizeof (struct pingpong_dest) * numofqps); + if (!my_dest){ + perror("malloc"); + return INVALID_SOCKET; + } + + rem_dest = malloc(sizeof (struct pingpong_dest) * numofqps ); + if (!rem_dest){ + perror("malloc"); + return INVALID_SOCKET; + } + + sockfd = servername ? pp_client_connect(servername, port) : + pp_server_connect(port); + + if (sockfd == INVALID_SOCKET) { + printf("pp_connect_sock(%s,%d) failed (%d)!\n", + servername, port, sockfd); + return INVALID_SOCKET; + } + + + for (i =0 ;ica_attr->p_port_attr[ib_port-1].lid; + my_dest[i].psn = rand() & 0xffffff; + if (!my_dest[i].lid) { + fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n"); + return 1; + } + my_dest[i].qpn = ctx->qp_attr[i].num; + /* TBD this should be changed inot VA and different key to each qp */ + my_dest[i].rkey = ctx->rkey; + my_dest[i].vaddr = (uintptr_t)ctx->buf + ctx->size; + + printf(" local address: LID %#04x, QPN %#06x, PSN %#06x, " + "RKey %#08x VAddr %#016Lx\n", + my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, + my_dest[i].rkey, my_dest[i].vaddr); + + rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]): + pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]); + if (rc) + return INVALID_SOCKET; + printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x, " + "RKey %#08x VAddr %#016Lx\n", + rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn, + rem_dest[i].rkey, rem_dest[i].vaddr); + + if (pp_connect_ctx(ctx, ib_port, my_dest[i].psn, &rem_dest[i], user_parm, i)) + return INVALID_SOCKET; + /* An additional handshake is required *after* moving qp to RTR. + Arbitrarily reuse exch_dest for this purpose. */ + rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]): + pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]); + if (rc) + return INVALID_SOCKET; + } + *p_rem_dest = rem_dest; + *p_my_dest = my_dest; + return sockfd; +} + +static void usage(const char *argv0) +{ + printf("Usage:\n"); + printf(" %s start a server and wait for connection\n", argv0); + printf(" %s connect to server at \n", argv0); + printf("\n"); + printf("Options:\n"); + printf(" -p, --port= listen on/connect to port (default 18515)\n"); + printf(" -c, --connection= connection type RC/UC (default RC)\n"); + printf(" -m, --mtu= mtu size (default 256)\n"); + printf(" -d, --ib-dev= use IB device (default first device found)\n"); + printf(" -i, --ib-port= use port of IB device (default 1)\n"); + printf(" -s, --size= size of message to exchange (default 1)\n"); + printf(" -t, --tx-depth= size of tx queue (default 50)\n"); + printf(" -n, --iters= number of exchanges (at least 2, default 1000)\n"); + printf(" -o, --outs= num of outstanding read/atom(default 4)\n"); + printf(" -a, --all Run sizes from 2 till 2^23\n"); + printf(" -C, --report-cycles report times in cpu cycle units (default microseconds)\n"); + printf(" -H, --report-histogram print out all results (default print summary only)\n"); + printf(" -U, --report-unsorted (implies -H) print out unsorted results (default sorted)\n"); + printf(" -V, --version display version number\n"); + printf(" -e, --events sleep on CQ events (default poll)\n"); +} + + + +static void print_report(struct report_options * options, + unsigned int iters, cycles_t *tstamp,int size) +{ + double cycles_to_units; + cycles_t median; + unsigned int i; + const char* units; + cycles_t *delta = malloc(iters * sizeof *delta); + + if (!delta) { + perror("malloc"); + return; + } + + for (i = 0; i < iters - 1; ++i) + delta[i] = tstamp[i + 1] - tstamp[i]; + + + if (options->cycles) { + cycles_to_units = 1; + units = "cycles"; + } else { + cycles_to_units = get_cpu_mhz()/1000000; + units = "usec"; + } + + if (options->unsorted) { + printf("#, %s\n", units); + for (i = 0; i < iters - 1; ++i) + printf("%d, %g\n", i + 1, delta[i] / cycles_to_units ); + } + + qsort(delta, iters - 1, sizeof *delta, cycles_compare); + + if (options->histogram) { + printf("#, %s\n", units); + for (i = 0; i < iters - 1; ++i) + printf("%d, %g\n", i + 1, delta[i] / cycles_to_units ); + } + + median = get_median(iters - 1, delta); + printf("%7d %d %7.2f %7.2f %7.2f\n", + size,iters,delta[0] / cycles_to_units , + delta[iters - 3] / cycles_to_units ,median / cycles_to_units ); + + free(delta); +} + +int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, + struct pingpong_dest *rem_dest, int size) +{ + ib_api_status_t ib_status; + int scnt, ccnt ; + ib_send_wr_t *bad_wr; + + ctx->list.vaddr = (uintptr_t) ctx->buf ; + ctx->list.length = size; + ctx->list.lkey = ctx->lkey; + ctx->wr.remote_ops.vaddr = rem_dest->vaddr; + ctx->wr.remote_ops.rkey = rem_dest->rkey; + ctx->wr.wr_id = PINGPONG_RDMA_WRID; + ctx->wr.ds_array = &ctx->list; + ctx->wr.num_ds = 1; + ctx->wr.wr_type = WR_RDMA_READ; + ctx->wr.send_opt = IB_SEND_OPT_SIGNALED; + ctx->wr.p_next = NULL; + + scnt = 0; + ccnt = 0; + + + /* Done with setup. Start the test. */ + + if(user_param->servername) { + while (scnt < user_param->iters ) { + + tstamp[scnt] = get_cycles(); + ib_status = ib_post_send(ctx->qp[0], &ctx->wr, &bad_wr); + if (ib_status != IB_SUCCESS) + { + fprintf(stderr, "Couldn't post send: scnt %d ccnt %d\n",scnt,ccnt); + return 1; + } + ++scnt; + PERF_DEBUG("scnt = %d \n",scnt); + + { + ib_wc_t wc; + ib_wc_t *p_wc_done,*p_wc_free; + + p_wc_free = &wc; + p_wc_done = NULL; + p_wc_free->p_next = NULL; + + do{ + ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done); + } while (ib_status == IB_NOT_FOUND); + + if (ib_status != IB_SUCCESS) { + fprintf(stderr, "Poll Send CQ failed %d\n", ib_status); + return 12; + } + + if (p_wc_done->status != IB_WCS_SUCCESS) { + fprintf(stderr, "Completion wth error at %s:\n", + user_param->servername ? "client" : "server"); + fprintf(stderr, "Failed status %d: wr_id %d syndrom 0x%x\n", + p_wc_done->status, (int) p_wc_done->wr_id, p_wc_done->vendor_specific); + return 1; + } + ++ccnt; + PERF_DEBUG("ccnt = %d \n",ccnt); + + } + + } + } + return(0); +} + + +int __cdecl main(int argc, char *argv[]) +{ + struct pingpong_context *ctx; + struct pingpong_dest *my_dest; + struct pingpong_dest *rem_dest; + struct user_parameters user_param; + char *ib_devname = NULL; + int port = 18515; + int ib_port = 1; + unsigned tmp_size,size = 2; + SOCKET sockfd = INVALID_SOCKET; + WSADATA wsaData; + int iResult; + int i = 0; + struct report_options report = {0}; + + + /* init default values to user's parameters */ + memset(&user_param, 0, sizeof(struct user_parameters)); + user_param.mtu = 0; /* signal choose default by device */ + user_param.iters = 1000; + user_param.tx_depth = 50; + user_param.servername = NULL; + user_param.connection_type = RC; + user_param.max_out_read = 4; /* the device capability on gen2 */ +/* Parameter parsing. */ + while (1) { + int c; + + static struct option long_options[] = { + { "port", 1, NULL, 'p' }, + { "mtu", 1, NULL, 'm' }, + { "outs", 1, NULL, 'o' }, + { "ib-dev", 1, NULL, 'd' }, + { "ib-port", 1, NULL, 'i' }, + { "size", 1, NULL, 's' }, + { "iters", 1, NULL, 'n' }, + { "tx-depth", 1, NULL, 't' }, + { "all", 0, NULL, 'a' }, + { "report-cycles", 0, NULL, 'C' }, + { "report-histogram", 0, NULL, 'H' }, + { "report-unsorted", 0, NULL, 'U' }, + { "version", 0, NULL, 'V' }, + { 0 } + }; + + + c = getopt_long(argc, argv, "p:m:d:i:s:n:t:aHUV", long_options, NULL); + if (c == -1) + break; + + switch (c) { + case 'p': + port = strtol(optarg, NULL, 0); + if (port < 0 || port > 65535) { + usage(argv[0]); + return 1; + } + break; + case 'm': + user_param.mtu = strtol(optarg, NULL, 0); + break; + case 'o': + user_param.max_out_read = strtol(optarg, NULL, 0); + break; + case 'a': + user_param.all = ALL; + break; + case 'V': + printf("read_lat version : %.2f\n",VERSION); + return 0; + break; + case 'd': + ib_devname = _strdup(optarg); + break; + + case 'i': + ib_port = strtol(optarg, NULL, 0); + if (ib_port < 0) { + usage(argv[0]); + return 2; + } + break; + + case 's': + size = strtol(optarg, NULL, 0); + if (size < 1) { + usage(argv[0]); return 3; + } + break; + + case 't': + user_param.tx_depth = strtol(optarg, NULL, 0); + if (user_param.tx_depth < 1) { + usage(argv[0]); return 4; + } + break; + + case 'n': + user_param.iters = strtol(optarg, NULL, 0); + if (user_param.iters < 2) { + usage(argv[0]); + return 5; + } + + break; + + case 'C': + report.cycles = 1; + break; + + case 'H': + report.histogram = 1; + break; + + case 'U': + report.unsorted = 1; + break; + + default: + usage(argv[0]); + return 5; + } + } + + if (optind == argc - 1) + user_param.servername = _strdup(argv[optind]); + else if (optind < argc) { + usage(argv[0]); + return 6; + } + + /* + * Done with parameter parsing. Perform setup. + */ + tstamp = malloc(user_param.iters * sizeof *tstamp); + if (!tstamp) { + perror("malloc"); + return 10; + } + printf("------------------------------------------------------------------\n"); + printf(" RDMA_Read Latency Test\n"); + + + // Initialize Winsock + iResult = WSAStartup(MAKEWORD(2,2), &wsaData); + if (iResult != NO_ERROR) { + printf("Error at WSAStartup()\n"); + return 1; + } + + tmp_size = size; + /* anyway make sure the connection is RC */ + if (user_param.all == ALL) { + /*since we run all sizes */ + size = 8388608; /*2^23 */ + } else if (size < 128) { + /* can cut up to 70 nsec probably related to cache line size */ + size = 128; + } + + srand(GetCurrentProcessId() * GetTickCount()); + + //TODO: get pagesize from sysinfo + page_size = 4096; + + //TODO:get the device names + + + ctx = pp_init_ctx(size,ib_port, &user_param); + if (!ctx) + return 8; + + + sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param); + if (sockfd == INVALID_SOCKET) + return 9; + + + printf("------------------------------------------------------------------\n"); + printf(" #bytes #iterations t_min[usec] t_max[usec] t_typical[usec]\n"); + /* For half duplex tests, server just waits for client to exit */ + /* use dummy my_dest struct*/ + if (!user_param.servername) { + pp_server_exch_dest(sockfd, my_dest,rem_dest); + send(sockfd, "done", sizeof "done",0); + closesocket(sockfd); + return 0; + } + + /* fix for true size in small msg size */ + if (tmp_size < 128) { + size = tmp_size ; + } + + if (user_param.all == ALL) { + for (i = 1; i < 24 ; ++i) { + size = 1 << i; + if(run_iter(ctx, &user_param, rem_dest, size)) + return 17; + print_report(&report, user_param.iters, tstamp, size); + } + } else { + if(run_iter(ctx, &user_param, rem_dest, size)) + return 18; + print_report(&report, user_param.iters, tstamp, size); + } + + pp_client_exch_dest(sockfd, my_dest,rem_dest); + send(sockfd, "done", sizeof "done",0); + closesocket(sockfd); + + printf("------------------------------------------------------------------\n"); + free(tstamp); + return 0; +} diff --git a/trunk/tools/perftests/user/read_lat/read_lat.rc b/trunk/tools/perftests/user/read_lat/read_lat.rc new file mode 100644 index 00000000..2dd3fb9e --- /dev/null +++ b/trunk/tools/perftests/user/read_lat/read_lat.rc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2005 Mellanox Technologies. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id: write_bw.rc 1611 2006-08-20 14:48:55Z sleybo $ + */ + + +#include + +#define VER_FILETYPE VFT_APP +#define VER_FILESUBTYPE VFT2_UNKNOWN + +#ifdef _DEBUG_ +#define VER_FILEDESCRIPTION_STR "RDMA read Latency Test (Debug)" +#else +#define VER_FILEDESCRIPTION_STR "RDMA read Latency Test " +#endif + +#define VER_INTERNALNAME_STR "ib_read_lat.exe" +#define VER_ORIGINALFILENAME_STR "ib_read_lat.exe" + +#include diff --git a/trunk/tools/perftests/user/send_lat/send_lat.c b/trunk/tools/perftests/user/send_lat/send_lat.c index 79e8126a..28bbf92b 100644 --- a/trunk/tools/perftests/user/send_lat/send_lat.c +++ b/trunk/tools/perftests/user/send_lat/send_lat.c @@ -39,9 +39,6 @@ #include "perf_defs.h" -#define SIGNAL 1 -#define MAX_INLINE 400 - static int page_size; cycles_t *tstamp; @@ -56,12 +53,6 @@ struct user_parameters { int tx_depth; }; -struct report_options { - int unsorted; - int histogram; - int cycles; /* report delta's in cycles, not microsec's */ -}; - void @@ -558,20 +549,6 @@ static void usage(const char *argv0) printf(" -e, --events sleep on CQ events (default poll)\n"); } -/* - * When there is an - * odd number of samples, the median is the middle number. - * even number of samples, the median is the mean of the - * two middle numbers. - * - */ -static inline cycles_t get_median(int n, cycles_t delta[]) -{ - if (n % 2) - return(delta[n / 2] + delta[n / 2 - 1]) / 2; - else - return delta[n / 2]; -} static void print_report(struct report_options * options, diff --git a/trunk/tools/perftests/user/write_bw/write_bw.c b/trunk/tools/perftests/user/write_bw/write_bw.c index 05a4e714..1931c683 100644 --- a/trunk/tools/perftests/user/write_bw/write_bw.c +++ b/trunk/tools/perftests/user/write_bw/write_bw.c @@ -37,8 +37,7 @@ #include "perf_defs.h" #include "get_clock.h" -#define RC 0 -#define UC 1 + struct user_parameters { const char *servername; @@ -237,7 +236,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user cq_create.pfn_comp_cb = pp_cq_comp_cb; ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq); if (ib_status != IB_SUCCESS) { - fprintf(stderr, "Couldn't create CQ\n"); + fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status); return NULL; } } @@ -284,6 +283,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user + memset(&qp_modify, 0, sizeof(ib_qp_mod_t)); qp_modify.req_state = IB_QPS_INIT; qp_modify.state.init.pkey_index = 0 ; @@ -363,12 +363,13 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, attr.state.rts.sq_psn = my_psn; if (user_parm->connection_type == RC) { - attr.state.rts.resp_res = 1; + attr.state.rts.init_depth = 1; attr.state.rts.local_ack_timeout = 14; attr.state.rts.retry_cnt = 7; attr.state.rts.rnr_retry_cnt = 7; attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT | IB_MOD_QP_RETRY_CNT | + IB_MOD_QP_INIT_DEPTH | IB_MOD_QP_LOCAL_ACK_TIMEOUT; } @@ -727,7 +728,7 @@ int __cdecl main(int argc, char *argv[]) ib_port = strtol(optarg, NULL, 0); if (ib_port < 0) { usage(argv[0]); - return 1; + return 2; } break; @@ -741,14 +742,16 @@ int __cdecl main(int argc, char *argv[]) case 't': user_param.tx_depth = strtol(optarg, NULL, 0); - if (user_param.tx_depth < 1) { usage(argv[0]); return 1; } + if (user_param.tx_depth < 1) { + usage(argv[0]); return 4; + } break; case 'n': user_param.iters = strtol(optarg, NULL, 0); if (user_param.iters < 2) { usage(argv[0]); - return 1; + return 5; } break; @@ -759,7 +762,7 @@ int __cdecl main(int argc, char *argv[]) default: usage(argv[0]); - return 1; + return 5; } } @@ -767,7 +770,7 @@ int __cdecl main(int argc, char *argv[]) user_param.servername = _strdup(argv[optind]); else if (optind < argc) { usage(argv[0]); - return 1; + return 6; } printf("------------------------------------------------------------------\n"); @@ -802,22 +805,20 @@ int __cdecl main(int argc, char *argv[]) if (user_param.all == ALL) { - /*since we run all sizes */ + /*since we run all sizes lets allocate big enough buffer */ size = 8388608; /*2^23 */ } - srand(GetCurrentProcessId() * GetTickCount()); //TODO: get pagesize from sysinfo page_size = 4096; - //TODO:get the device names + //TODO get the device names ctx = pp_init_ctx(size, ib_port, &user_param); if (!ctx) - return 1; - + return 8; sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param); if (sockfd == INVALID_SOCKET) return 9; diff --git a/trunk/tools/perftests/user/write_lat/write_lat.c b/trunk/tools/perftests/user/write_lat/write_lat.c index fe1c1280..329cbe28 100644 --- a/trunk/tools/perftests/user/write_lat/write_lat.c +++ b/trunk/tools/perftests/user/write_lat/write_lat.c @@ -39,12 +39,7 @@ #include "get_clock.h" -#define RC 0 -#define UC 1 - -static int page_size; -cycles_t *tstamp; struct user_parameters { const char *servername; int connection_type; @@ -54,11 +49,10 @@ struct user_parameters { int tx_depth; }; -struct report_options { - int unsorted; - int histogram; - int cycles; /* report delta's in cycles, not microsec's */ -}; +static int page_size; + +cycles_t *tstamp; + void @@ -232,15 +226,11 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user cq_create.pfn_comp_cb = pp_cq_comp_cb; ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq); if (ib_status != IB_SUCCESS) { - fprintf(stderr, "Couldn't create CQ\n"); + fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status); return NULL; } } - - - - { ib_qp_create_t qp_create; @@ -279,6 +269,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user + memset(&qp_modify, 0, sizeof(ib_qp_mod_t)); qp_modify.req_state = IB_QPS_INIT; qp_modify.state.init.pkey_index = 0 ; @@ -302,13 +293,6 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user fprintf(stderr, "max inline size %d\n",ctx->qp_attr[0].sq_max_inline); } - - ctx->wr.wr_id = PINGPONG_RDMA_WRID; - ctx->wr.ds_array = &ctx->list; - ctx->wr.num_ds = 1; - ctx->wr.wr_type = WR_RDMA_WRITE; - ctx->wr.p_next = NULL; - return ctx; } @@ -316,7 +300,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, - struct pingpong_dest *dest, struct user_parameters *user_parm,int index) + struct pingpong_dest *dest, struct user_parameters *user_parm,int qpindex) { ib_api_status_t ib_status; @@ -356,7 +340,7 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, IB_MOD_QP_PRIMARY_AV; - ib_status = ib_modify_qp(ctx->qp[index], &attr); + ib_status = ib_modify_qp(ctx->qp[0], &attr); if(ib_status != IB_SUCCESS){ fprintf(stderr, "Failed to modify QP to RTR\n"); return 1; @@ -367,16 +351,17 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, attr.state.rts.sq_psn = my_psn; if (user_parm->connection_type == RC) { - attr.state.rts.resp_res = 1; + attr.state.rts.init_depth = 1; attr.state.rts.local_ack_timeout = 14; attr.state.rts.retry_cnt = 7; attr.state.rts.rnr_retry_cnt = 7; attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT | IB_MOD_QP_RETRY_CNT | + IB_MOD_QP_INIT_DEPTH | IB_MOD_QP_LOCAL_ACK_TIMEOUT; - } - ib_status = ib_modify_qp(ctx->qp[index], &attr); + } + ib_status = ib_modify_qp(ctx->qp[0], &attr); if(ib_status != IB_SUCCESS){ fprintf(stderr, "Failed to modify QP to RTS\n"); return 1; @@ -490,20 +475,6 @@ static void usage(const char *argv0) printf(" -V, --version display version number\n"); } -/* - * When there is an - * odd number of samples, the median is the middle number. - * even number of samples, the median is the mean of the - * two middle numbers. - * - */ -static inline cycles_t get_median(int n, cycles_t delta[]) -{ - if (n % 2) - return(delta[n / 2] + delta[n / 2 - 1]) / 2; - else - return delta[n / 2]; -} static void print_report(struct report_options * options, @@ -560,17 +531,13 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, struct pingpong_dest *rem_dest, int size) { ib_api_status_t ib_status; - ib_qp_handle_t qp; + int scnt, ccnt, rcnt; ib_send_wr_t *bad_wr; volatile char *poll_buf; volatile char *post_buf; - int scnt, ccnt, rcnt; - int iters; - int tx_depth; - iters = user_param->iters; - tx_depth = user_param->tx_depth; + ctx->list.vaddr = (uintptr_t) ctx->buf ; @@ -578,6 +545,10 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, ctx->list.lkey = ctx->lkey; ctx->wr.remote_ops.vaddr = rem_dest->vaddr; ctx->wr.remote_ops.rkey = rem_dest->rkey; + ctx->wr.wr_id = PINGPONG_RDMA_WRID; + ctx->wr.ds_array = &ctx->list; + ctx->wr.num_ds = 1; + ctx->wr.wr_type = WR_RDMA_WRITE; if ((uint32_t)size > ctx->qp_attr[0].sq_max_inline) {/* complaince to perf_main */ ctx->wr.send_opt = IB_SEND_OPT_SIGNALED; @@ -595,10 +566,9 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, poll_buf = ctx->poll_buf; post_buf = ctx->post_buf; } - qp = ctx->qp[0]; /* Done with setup. Start the test. */ - while (scnt < iters || ccnt < iters || rcnt < iters) { + while (scnt < user_param->iters || ccnt < user_param->iters || rcnt < user_param->iters) { /* Wait till buffer changes. */ if (rcnt < user_param->iters && !(scnt < 1 && user_param->servername)) { @@ -615,7 +585,7 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, tstamp[scnt] = get_cycles(); *post_buf = (char)++scnt; - ib_status = ib_post_send(qp, &ctx->wr, &bad_wr); + ib_status = ib_post_send(ctx->qp[0], &ctx->wr, &bad_wr); if (ib_status != IB_SUCCESS) { fprintf(stderr, "Couldn't post send:scnt %d ccnt=%d \n",scnt,ccnt); @@ -662,10 +632,6 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param, int __cdecl main(int argc, char *argv[]) { - - - - struct pingpong_context *ctx; struct pingpong_dest *my_dest; struct pingpong_dest *rem_dest; @@ -692,19 +658,19 @@ int __cdecl main(int argc, char *argv[]) int c; static struct option long_options[] = { - { "port", 1, NULL, 'p' }, - { "connection", 1, NULL, 'c' }, - { "mtu", 1, NULL, 'm' }, - { "ib-dev", 1, NULL, 'd' }, - { "ib-port", 1, NULL, 'i' }, - { "size", 1, NULL, 's' }, - { "iters", 1, NULL, 'n' }, - { "tx-depth", 1, NULL, 't' }, - { "all", 0, NULL, 'a' }, - { "report-cycles", 0, NULL, 'C' }, - { "report-histogram", 0, NULL, 'H' }, - { "report-unsorted", 0, NULL, 'U' }, - { "version", 0, NULL, 'V' }, + { "port", 1, NULL, 'p' }, + { "connection", 1, NULL, 'c' }, + { "mtu", 1, NULL, 'm' }, + { "ib-dev", 1, NULL, 'd' }, + { "ib-port", 1, NULL, 'i' }, + { "size", 1, NULL, 's' }, + { "iters", 1, NULL, 'n' }, + { "tx-depth", 1, NULL, 't' }, + { "all", 0, NULL, 'a' }, + { "report-cycles", 0, NULL, 'C' }, + { "report-histogram", 0, NULL, 'H' }, + { "report-unsorted", 0, NULL, 'U' }, + { "version", 0, NULL, 'V' }, { 0 } }; @@ -722,8 +688,7 @@ int __cdecl main(int argc, char *argv[]) break; case 'c': if (strcmp("UC",optarg)==0) - user_param.connection_type=1; - /* default is 0 for any other option RC*/ + user_param.connection_type=UC; break; case 'm': @@ -807,14 +772,13 @@ int __cdecl main(int argc, char *argv[]) } printf("------------------------------------------------------------------\n"); printf(" RDMA_Write Latency Test\n"); - printf("Inline data is used up to 400 bytes message\n"); if (user_param.connection_type==0) { printf("Connection type : RC\n"); } else { printf("Connection type : UC\n"); } - /* Done with parameter parsing. Perform setup. */ + // Initialize Winsock iResult = WSAStartup(MAKEWORD(2,2), &wsaData); @@ -842,6 +806,7 @@ int __cdecl main(int argc, char *argv[]) sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param); if (sockfd == INVALID_SOCKET) return 9; + printf("------------------------------------------------------------------\n"); printf(" #bytes #iterations t_min[usec] t_max[usec] t_typical[usec]\n"); -- 2.41.0