[tests] add option to run ib_send_bw with events. (mlnx 3032)

git-svn-id: svn://openib.tc.cornell.edu/gen1@1503 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/tools/perftests/user/perf_defs.h b/trunk/tools/perftests/user/perf_defs.h
index 72693d204a236d8439e14c477b050f484bd5f3bc..2b29ae15f406fd8042b18c0aa0c2ac2be668bdb3 100644 (file)
--- a/trunk/tools/perftests/user/perf_defs.h
+++ b/trunk/tools/perftests/user/perf_defs.h
@@ -114,6 +114,7 @@ struct pingpong_context {
        volatile char           *poll_buf;
 
        int                             *scnt,*ccnt;
+       cl_waitobj_handle_t             cq_waitobj;
 };
 
 

diff --git a/trunk/tools/perftests/user/send_bw/send_bw.c b/trunk/tools/perftests/user/send_bw/send_bw.c
index eca89bed5291dbc1b99b6151b2806c0edd1d7291..25e5201c2b020105a82aeb74cd9dfafea39be53b 100644 (file)
--- a/trunk/tools/perftests/user/send_bw/send_bw.c
+++ b/trunk/tools/perftests/user/send_bw/send_bw.c
@@ -51,7 +51,7 @@ struct user_parameters {
        int                     iters;
        int                     tx_depth;
        int                     duplex;
-    int use_event;
+       int                     use_event;
 };
 
 static int page_size;
@@ -62,7 +62,7 @@ int post_recv;
 
 void
 pp_cq_comp_cb(
-       IN              const   ib_cq_handle_t                          h_cq,
+       IN              const   ib_cq_handle_t                          h_cq,
        IN                              void                                            *cq_context )
 {
        UNUSED_PARAM( h_cq );
@@ -72,7 +72,8 @@ pp_cq_comp_cb(
 
 
 
-static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_parameters *user_parm)
+static struct pingpong_context *pp_init_ctx(unsigned size,int port, 
+                                       struct user_parameters *user_parm)
 {
 
        struct pingpong_context *ctx;
@@ -101,14 +102,14 @@ static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_
        ctx->tx_depth = user_parm->tx_depth;
        /* in case of UD need space for the GRH */
        if (user_parm->connection_type==UD) {
-               ctx->buf = malloc(( size + 40 ) * 2); //PORTED ALINGED
+               ctx->buf = malloc(( size + 40 ) * 2);
                if (!ctx->buf) {
                        fprintf(stderr, "Couldn't allocate work buf.\n");
                        return NULL;
                }
                memset(ctx->buf, 0, ( size + 40 ) * 2);
        } else {
-               ctx->buf = malloc( size * 2); //PORTED ALINGED
+               ctx->buf = malloc( size * 2);
                if (!ctx->buf) {
                        fprintf(stderr, "Couldn't allocate work buf.\n");
                        return NULL;
@@ -199,16 +200,8 @@ static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_
                        }
                }
        }
-               
-       if (user_parm->use_event) {
-//PORTED               ctx->channel = ibv_create_comp_channel(ctx->context);
-               ctx->channel = NULL;//remove when PORTED
-               if (!ctx->channel) {
-                       fprintf(stderr, "Couldn't create completion channel\n");
-                       return NULL;
-               }
-       } else
-               ctx->channel = NULL;                  
+
+       ctx->channel = NULL;
 
        ib_status = ib_alloc_pd(ctx->ca ,
                                                IB_PDT_NORMAL,
@@ -241,16 +234,42 @@ static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_
                        return NULL;
                }
 
+               if (user_parm->use_event) {
+                       cl_status_t cl_status;
+
+                       cl_status = cl_waitobj_create( FALSE, &ctx->cq_waitobj );
+                       if( cl_status != CL_SUCCESS ) {
+                               ctx->cq_waitobj = NULL;
+                               fprintf(stderr, "cl_waitobj_create() returned %s\n", CL_STATUS_MSG(cl_status) );
+                               return NULL;
+                       }
+
+                       cq_create.h_wait_obj = ctx->cq_waitobj;
+                       cq_create.pfn_comp_cb = NULL;
+               } else {
+                       cq_create.h_wait_obj = NULL;
+                       cq_create.pfn_comp_cb = pp_cq_comp_cb;
+               }
+
                cq_create.size = user_parm->tx_depth*2;
-               cq_create.h_wait_obj = NULL;
-               cq_create.pfn_comp_cb = pp_cq_comp_cb;
                ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq);
                if (ib_status != IB_SUCCESS) {
                        fprintf(stderr, "Couldn't create CQ\n");
+                       fprintf(stderr, "ib_status = %d\n", ib_status);
                        return NULL;
                }
+
+               if (user_parm->use_event) {
+                       ib_status = ib_rearm_cq( ctx->scq, FALSE );
+                       if( ib_status )
+                       {
+                               ib_destroy_cq( ctx->scq, NULL );
+                               fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str( ib_status ));
+                               return NULL;
+                       }
+               }
        }
-       
+
        {
                ib_qp_create_t  qp_create;
                memset(&qp_create, 0, sizeof(ib_qp_create_t));
@@ -579,7 +598,7 @@ int run_iter_bi(struct pingpong_context *ctx, struct user_parameters *user_param
                struct pingpong_dest *rem_dest, int size)
 {
 
-       ib_qp_handle_t  qp;
+       ib_qp_handle_t  qp;
        int                             scnt, ccnt, rcnt;
        ib_recv_wr_t            *bad_wr_recv;
        ib_api_status_t ib_status;
@@ -648,24 +667,19 @@ int run_iter_bi(struct pingpong_context *ctx, struct user_parameters *user_param
                        p_wc_free = &wc;
                        p_wc_free->p_next = NULL;
                        p_wc_done = NULL;
-#if PORTED
+
                        if (user_param->use_event) {
-                               struct ibv_cq *ev_cq;
-                               void          *ev_ctx;
-                               if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
-                                       fprintf(stderr, "Failed to get cq_event\n");
-                                       return 1;
-                               }                
-                               if (ev_cq != ctx->cq) {
-                                       fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
-                                       return 1;
-                               }
-                               if (ibv_req_notify_cq(ctx->cq, 0)) {
-                                       fprintf(stderr, "Couldn't request CQ notification\n");
+                               cl_status_t     cl_status;
+
+                               PERF_DEBUG("%s:%d IN cl_waitobj_wait_on", __FUNCTION__, __LINE__);
+                               cl_status = cl_waitobj_wait_on( ctx->cq_waitobj, EVENT_NO_TIMEOUT, TRUE );
+                               if( cl_status != CL_SUCCESS )
+                               {
+                                       fprintf(stderr, "cl_waitobj_wait_on() (%d)\n", cl_status);
                                        return 1;
                                }
-                       }
-#endif
+                       } 
+
                        do {
                                ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
                                if (ib_status == IB_SUCCESS ) {
@@ -678,6 +692,7 @@ int run_iter_bi(struct pingpong_context *ctx, struct user_parameters *user_param
                                                scnt, ccnt);
                                                return 1;
                                        }
+
                                        switch ((int) p_wc_done->wr_id) {
                                        case PINGPONG_SEND_WRID:
                                                tcompleted[ccnt] = get_cycles();
@@ -716,6 +731,15 @@ int run_iter_bi(struct pingpong_context *ctx, struct user_parameters *user_param
                                return 1;
                        }
 
+                       if (user_param->use_event) {
+                               ib_status = ib_rearm_cq( ctx->scq, FALSE );
+                               if( ib_status )
+                               {
+                                       ib_destroy_cq( ctx->scq, NULL );
+                                       fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str( ib_status ));
+                                       return 1;
+                               }
+                       }
                }
        }
        return(0);
@@ -726,7 +750,7 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para
                 struct pingpong_dest *rem_dest, int size)
 {
 
-       ib_qp_handle_t  qp;
+       ib_qp_handle_t  qp;
        int                             scnt, ccnt, rcnt;
        ib_recv_wr_t            *bad_wr_recv;
        ib_api_status_t ib_status;
@@ -770,25 +794,21 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para
                        p_wc_free->p_next = NULL;
                        
                        /*Server is polling on recieve first */
-#if PORTED
                        if (user_param->use_event) {
-                               struct ibv_cq *ev_cq;
-                               void          *ev_ctx;
-                               if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
-                                       fprintf(stderr, "Failed to get cq_event\n");
-                                       return 1;
-                               }                
-                               if (ev_cq != ctx->cq) {
-                                       fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
+                               cl_status_t     cl_status;
+
+                               PERF_DEBUG("%s:%d IN cl_waitobj_wait_on\n", __FUNCTION__, __LINE__);
+                               cl_status = cl_waitobj_wait_on( ctx->cq_waitobj, EVENT_NO_TIMEOUT, TRUE );
+                               if( cl_status != CL_SUCCESS )
+                               {
+                                       fprintf(stderr, "cl_waitobj_wait_on() (%d)\n", cl_status);
                                        return 1;
                                }
-                               if (ibv_req_notify_cq(ctx->cq, 0)) {
-                               fprintf(stderr, "Couldn't request CQ notification\n");
-                               return 1;
-                               }
-                       }
-#endif
+                               PERF_DEBUG("%s:%d OUT cl_waitobj_wait_on\n", __FUNCTION__, __LINE__);
+                       } 
+
                        do {
+
                                ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
                                if (ib_status == IB_SUCCESS) {
                                        tcompleted[ccnt] = get_cycles();
@@ -818,20 +838,29 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para
                                        p_wc_free->p_next = NULL;
                                        p_wc_done = NULL;
                                }
-                               
-                               
                        } while (ib_status == IB_SUCCESS);
                        if (ib_status != IB_NOT_FOUND) {
                                fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status);
                                return 12;
                        }
 
+                       if (user_param->use_event) {
+                               ib_status = ib_rearm_cq( ctx->scq, FALSE );
+                               if( ib_status )
+                               {
+                                       ib_destroy_cq( ctx->scq, NULL );
+                                       fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str( ib_status ));
+                                       return 1;
+                               }
+                       }
+
                }
        } else {
                /* client is posting and not receiving. */
                while (scnt < user_param->iters || ccnt < user_param->iters) {
                        while (scnt < user_param->iters && (scnt - ccnt) < user_param->tx_depth ) {
                                ib_send_wr_t    *bad_wr;
+
                                tposted[scnt] = get_cycles();
                                ib_status = ib_post_send(qp, &ctx->wr, &bad_wr);
                                if (ib_status != IB_SUCCESS) {
@@ -850,25 +879,20 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para
                                p_wc_free = &wc;
                                p_wc_free->p_next = NULL;
                                p_wc_done = NULL;
-                               
-#if PORTED
-                               if (user_param->use_event) {
-                                       struct ibv_cq *ev_cq;
-                                       void          *ev_ctx;
-                                       if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
-                                               fprintf(stderr, "Failed to get cq_event\n");
-                                               return 1;
-                                       }                
-                                       if (ev_cq != ctx->cq) {
-                                               fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
-                                               return 1;
-                                       }
-                                       if (ibv_req_notify_cq(ctx->cq, 0)) {
-                                               fprintf(stderr, "Couldn't request CQ notification\n");
+
+                               if ( (user_param->use_event) ) {
+                                       cl_status_t     cl_status;
+
+                                       PERF_DEBUG("%s:%d IN cl_waitobj_wait_on\n", __FUNCTION__, __LINE__);
+                                       cl_status = cl_waitobj_wait_on( ctx->cq_waitobj, EVENT_NO_TIMEOUT, TRUE );
+                                       if( cl_status != CL_SUCCESS )
+                                       {
+                                               fprintf(stderr, "cl_waitobj_wait_on() (%d)\n", cl_status);
                                                return 1;
                                        }
+                                       PERF_DEBUG("%s:%d OUT cl_waitobj_wait_on\n", __FUNCTION__, __LINE__);
                                } 
-#endif
+
                                do {
                                        ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
                                        if (ib_status == IB_SUCCESS ) {
@@ -887,13 +911,22 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para
                                                p_wc_free->p_next = NULL;
                                                p_wc_done = NULL;
                                        }
-
-                               } while (ib_status == IB_SUCCESS );
+                                       } while (ib_status == IB_SUCCESS );
                                if (ib_status != IB_NOT_FOUND) {
                                        fprintf(stderr, "poll CQ failed %d\n", ib_status);
                                        return 1;
                                }
 
+                               if ( (user_param->use_event) ) {
+                                       ib_status = ib_rearm_cq( ctx->scq, FALSE );
+                                       if( ib_status )
+                                       {
+                                               ib_destroy_cq( ctx->scq, NULL );
+                                               fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str( ib_status ));
+                                               return 1;
+                                       }
+                               }
+
                                PERF_DEBUG("ccnt = %d \n",ccnt);
                        }
                }
@@ -904,21 +937,20 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para
 
 int __cdecl main(int argc, char *argv[])
 {
-       struct pingpong_context *ctx;
-       struct pingpong_dest     *my_dest;
-       struct pingpong_dest    *rem_dest;
-       struct user_parameters  user_param;
+       struct pingpong_context         *ctx;
+       struct pingpong_dest            *my_dest;
+       struct pingpong_dest            *rem_dest;
+       struct user_parameters          user_param;
        char                            *ib_devname = NULL;
        int                             port = 18515;
        int                             ib_port = 1;
        unsigned                        size = 65536;
-       SOCKET                  sockfd = INVALID_SOCKET;
+       SOCKET                          sockfd = INVALID_SOCKET;
        int                             i = 0;
        int                             size_max_pow = 24;
-       WSADATA         wsaData;
+       WSADATA                         wsaData;
        int                             iResult;
 
-       
 
        /* init default values to user's parameters */
        memset(&user_param, 0, sizeof(struct user_parameters));
@@ -985,7 +1017,7 @@ int __cdecl main(int argc, char *argv[])
                        break;
                case 'i':
                        ib_port = strtol(optarg, NULL, 0);
-                       if (ib_port < 0) {
+                       if (ib_port <= 0) {
                                usage(argv[0]);
                                return 1;
                        }
@@ -1081,15 +1113,10 @@ int __cdecl main(int argc, char *argv[])
        if (sockfd == INVALID_SOCKET)
                return 9;
        
-#if PORTED
        if (user_param.use_event) {
                printf("Test with events.\n");
-               if (ibv_req_notify_cq(ctx->cq, 0)) {
-                       fprintf(stderr, "Couldn't request CQ notification\n");
-                       return 1;
-               } 
        }
-#endif
+
        printf("------------------------------------------------------------------\n");
        printf(" #bytes #iterations    BW peak[MB/sec]    BW average[MB/sec]  \n");
 
@@ -1164,7 +1191,18 @@ int __cdecl main(int argc, char *argv[])
        goto end;
 
 
-end:   
+end:
+    if (user_param.use_event) {
+               cl_status_t cl_status;
+
+               cl_status = cl_waitobj_destroy( ctx->cq_waitobj );
+               if( cl_status != CL_SUCCESS )
+               {
+                       fprintf (stderr,
+                               "cl_waitobj_destroy() returned %s\n", CL_STATUS_MSG(cl_status));
+               }
+       }
+
        WSACleanup();
        return 0;
 }