r3525: Improve dtest and measure RDMA reads

Signed-off by: Arlin Davis <ardavis@ichips.intel.com>
Signed-off-by: James Lentini <jlentini@netapp.com>

diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
index 0b8ed29199da953053da3a151d27d8d0509a9b63..b7856141a8be92acfc6c9a8bbbded13d5c8a6d68 100644 (file)
--- a/test/dtest/dtest.c
+++ b/test/dtest/dtest.c
@@ -42,10 +42,12 @@
 #include <getopt.h>
 
 #ifndef DAPL_PROVIDER
-#define DAPL_PROVIDER "OpenIB1_2"
+#define DAPL_PROVIDER "OpenIB-ib0"
 #endif
 
 #define MAX_POLLING_CNT 50000
+#define MAX_RDMA_RD    4
+#define MAX_PROCS      1000
 
 /* Header files needed for DAT/uDAPL */
 #include    "dat/udat.h"
@@ -66,7 +68,8 @@ static DAT_PSP_HANDLE     h_psp = DAT_HANDLE_NULL;
 static DAT_CR_HANDLE      h_cr = DAT_HANDLE_NULL;
 
 static DAT_EVD_HANDLE     h_async_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE     h_dto_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE     h_dto_req_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE     h_dto_rcv_evd = DAT_HANDLE_NULL;
 static DAT_EVD_HANDLE     h_cr_evd = DAT_HANDLE_NULL;
 static DAT_EVD_HANDLE     h_conn_evd = DAT_HANDLE_NULL;
 static DAT_CNO_HANDLE     h_dto_cno = DAT_HANDLE_NULL;
@@ -123,7 +126,8 @@ struct {
     double  epc;
     double  epf;
     double  rdma_wr;
-    double  rdma_rd;
+    double  rdma_rd[MAX_RDMA_RD];
+    double  rdma_rd_total;
     double  rtt;
     double  close;
 } time;
@@ -137,7 +141,7 @@ static int  verbose=0;
 static int  polling=1;
 static int  poll_count=0;
 static int  rdma_wr_poll_count=0;
-static int  rdma_rd_poll_count=0;
+static int  rdma_rd_poll_count[MAX_RDMA_RD]={0};
 static int  pin_memory=0;
 static int  delay=0;
 static int  buf_len=RDMA_BUFFER_SIZE;
@@ -147,9 +151,6 @@ static int  recv_msg_index=0;
 static int  burst_msg_posted=0;
 static int  burst_msg_index=0;
 
-#define MAX_RDMA_RD    4
-#define MAX_PROCS      1000
-
 static pid_t   child[MAX_PROCS+1];
 
 /* forward prototypes */
@@ -181,7 +182,7 @@ DAT_RETURN     do_ping_pong_msg( void );
 
 main(int argc, char **argv)
 {
-       int c;
+       int i,c;
        DAT_RETURN  ret;
 
        /* parse arguments */
@@ -191,7 +192,6 @@ main(int argc, char **argv)
                {
                        case 's':
                                server = 1;
-                               printf("%d Running as server\n",getpid());
                                fflush(stdout);
                                break;
                        case 'c':
@@ -310,7 +310,8 @@ main(int argc, char **argv)
        ep_attr.ep_provider_specific        = NULL;
 
        start = get_time();
-       ret = dat_ep_create( h_ia, h_pz, h_dto_evd, h_dto_evd, h_conn_evd, &ep_attr, &h_ep );
+       ret = dat_ep_create( h_ia, h_pz, h_dto_rcv_evd, 
+                           h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep );
        stop = get_time();
        time.epc += ((stop - start)*1.0e6);
        time.total += time.epc;
@@ -432,13 +433,16 @@ cleanup:
                LOGPRINTF("%d Closed Interface Adaptor\n",getpid());
 
         printf( "\n%d: DAPL Test Complete.\n\n",getpid());
-       printf( "%d: RDMA write:  Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n",
-               getpid(), time.rdma_wr, burst, time.rdma_wr/burst, rdma_wr_poll_count );
-       printf( "%d: RDMA read:   Total=%10.2lf usec,   %d bursts, itime=%10.2lf usec, pc=%d\n",
-               getpid(), time.rdma_rd, MAX_RDMA_RD, time.rdma_rd/MAX_RDMA_RD, rdma_rd_poll_count );
-       printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n\n",
-               getpid(), time.rtt, burst, time.rtt/burst, poll_count );
-
+       printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n", 
+               getpid(), time.rtt, burst, time.rtt/burst, poll_count );
+       printf( "%d: RDMA write:  Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n", 
+               getpid(), time.rdma_wr, burst, 
+               time.rdma_wr/burst, rdma_wr_poll_count );
+       for(i=0;i<MAX_RDMA_RD;i++) {
+           printf( "%d: RDMA read:   Total=%10.2lf usec,   %d bursts, itime=%10.2lf usec, pc=%d\n", 
+                  getpid(),time.rdma_rd_total,MAX_RDMA_RD,
+                  time.rdma_rd[i],rdma_rd_poll_count[i] );
+       }
         printf( "%d: open:      %10.2lf usec\n", getpid(), time.open  );
         printf( "%d: close:     %10.2lf usec\n", getpid(), time.close );
         printf( "%d: PZ create: %10.2lf usec\n", getpid(), time.pzc );
@@ -509,7 +513,7 @@ send_msg(   void                    *data,
     if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
        if ( polling ) {
            printf("%d Polling post send completion...\n",getpid());
-           while (  dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY );
+           while (  dat_evd_dequeue( h_dto_req_evd, &event ) == DAT_QUEUE_EMPTY );
        }
        else {
            LOGPRINTF("%d waiting for post_send completion event\n", getpid());
@@ -517,15 +521,15 @@ send_msg(   void                    *data,
                DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
                ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
                LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-               if ( evd != h_dto_evd ) {
+               if ( evd != h_dto_req_evd ) {
                    fprintf(stderr,
-                       "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
+                       "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
                        getpid());
                    return( DAT_ABORT );
                }
            }
            /* use wait to dequeue */
-           ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+           ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
            if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
                        getpid(),DT_RetToString(ret));
@@ -790,7 +794,7 @@ connect_ep( char *hostname, int conn_id )
      */
     if ( polling ) {
        printf("%d Polling for remote to send RMR data\n",getpid());
-       while (  dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY );
+       while (  dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
     }
     else  {
        printf("%d Waiting for remote to send RMR data\n",getpid());
@@ -799,22 +803,22 @@ connect_ep( char *hostname, int conn_id )
            DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
            ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
            LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-           if ( evd != h_dto_evd ) {
+           if ( evd != h_dto_rcv_evd ) {
                fprintf(stderr,
-                       "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
+                       "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
                        getpid());
                return( DAT_ABORT );
            }
        }
        /* use wait to dequeue */
-       ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+       ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error waiting on h_dto_evd: %s\n",
+           fprintf(stderr, "%d Error waiting on h_dto_rcv_evd: %s\n",
                    getpid(),DT_RetToString(ret));
            return(ret);
        }
        else {
-           LOGPRINTF("%d dat_evd_wait h_dto_evd completed\n", getpid());
+           LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid());
        }
     }
 
@@ -988,7 +992,7 @@ do_rdma_write_with_msg( )
         *  Collect first event, write completion or the inbound recv with immed
         */
        if ( polling ) {
-           while (  dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY )
+           while (  dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )
                rdma_wr_poll_count++;
        }
        else {
@@ -997,14 +1001,15 @@ do_rdma_write_with_msg( )
                    DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
                    ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
                    LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-                   if ( evd != h_dto_evd ) {
-                           fprintf(stderr, "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
-                                           getpid());
+                   if ( evd != h_dto_rcv_evd ) {
+                           fprintf(stderr, 
+                                  "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+                                   getpid());
                            return( ret );
                    }
            }
            /* use wait to dequeue */
-           ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+           ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
            if (ret != DAT_SUCCESS) {
                    fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
                                            getpid(),DT_RetToString(ret));
@@ -1067,8 +1072,6 @@ do_rdma_read_with_msg( )
 
        printf("\n %d RDMA READ DATA with SEND MSG\n\n",getpid());
 
-       cookie.as_64 = 0x5555;
-
        if ( recv_msg_index >= MSG_BUF_COUNT )
                return( DAT_ABORT );
 
@@ -1087,22 +1090,71 @@ do_rdma_read_with_msg( )
        l_iov.pad             = 0;
        l_iov.virtual_address = (DAT_VADDR)(unsigned long)sbuf;
        l_iov.segment_length  = buf_len;
-
-       start = get_time();
+       
        for (i=0;i<MAX_RDMA_RD;i++) {
-           cookie.as_64 = 0x9999;
-           ret = dat_ep_post_rdma_read(    h_ep,               // ep_handle
-                                           1,                  // num_segments
-                                           &l_iov,             // LMR
-                                           cookie,             // user_cookie
-                                           &r_iov,             // RMR
-                                           DAT_COMPLETION_SUPPRESS_FLAG );
-           if (ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n",
-                                       getpid(),DT_RetToString(ret));
-               return( DAT_ABORT );
-           }
-           LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);
+           cookie.as_64 = 0x9999;
+           start = get_time();
+           ret = dat_ep_post_rdma_read(    h_ep,               // ep_handle
+                                           1,                  // num_segments
+                                           &l_iov,             // LMR
+                                           cookie,             // user_cookie
+                                           &r_iov,             // RMR
+                                           DAT_COMPLETION_DEFAULT_FLAG );
+           if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n", 
+                                       getpid(),DT_RetToString(ret));
+               return( DAT_ABORT );
+           }
+           
+           if (polling) {
+               while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY)
+                       rdma_rd_poll_count[i]++;
+           } 
+           else {
+               LOGPRINTF("%d waiting for rdma_read completion event\n", getpid());
+               if (use_cno) {
+                       DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+                       ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+                       LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
+                       if ( evd != h_dto_req_evd ) {
+                               fprintf(stderr, 
+                               "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n", 
+                               getpid());
+                               return( DAT_ABORT );
+                       }
+               }
+               /* use wait to dequeue */
+               ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n", 
+                               getpid(),DT_RetToString(ret));
+                       return ret;
+               }
+           }
+           /* validate event number, len, cookie, and status */
+           if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+               fprintf(stderr, "%d: ERROR: DTO event number %s\n", 
+                       getpid(),DT_EventToSTr(event.event_number));
+               return( DAT_ABORT );
+           }
+           if ((event.event_data.dto_completion_event_data.transfered_length != buf_len ) ||
+               (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0x9999 )) {
+               fprintf(stderr, "%d: ERROR: DTO len %d or cookie %x\n", 
+                       getpid(),
+                       event.event_data.dto_completion_event_data.transfered_length,
+                       event.event_data.dto_completion_event_data.user_cookie.as_64 );
+               return( DAT_ABORT );
+           }
+           if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
+               fprintf(stderr, "%d: ERROR: DTO event status %s\n", 
+                       getpid(),DT_RetToString(ret));
+               return( DAT_ABORT );
+           }
+           stop = get_time();
+           time.rdma_rd[i] = ((stop - start)*1.0e6);
+           time.rdma_rd_total += time.rdma_rd[i];
+
+           LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);
        }
 
        /*
@@ -1134,8 +1186,7 @@ do_rdma_read_with_msg( )
         */
        printf("%d Waiting for inbound message....\n",getpid());
        if ( polling ) {
-           while (  dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY )
-               rdma_rd_poll_count++;
+           while (  dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
        }
        else {
            LOGPRINTF("%d waiting for message receive event\n", getpid());
@@ -1143,22 +1194,21 @@ do_rdma_read_with_msg( )
                    DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
                    ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
                    LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-                   if ( evd != h_dto_evd ) {
-                           fprintf(stderr, "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
-                                           getpid());
+                   if ( evd != h_dto_rcv_evd ) {
+                           fprintf(stderr, 
+                                  "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+                                   getpid());
                            return( ret );
                    }
            }
            /* use wait to dequeue */
-           ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+           ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
            if (ret != DAT_SUCCESS) {
                    fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
                                            getpid(),DT_RetToString(ret));
                    return( ret );
            }
        }
-       stop = get_time();
-       time.rdma_rd = ((stop - start)*1.0e6);
 
        /* validate event number and status */
        printf("%d inbound rdma_read; send message arrived!\n",getpid());
@@ -1283,7 +1333,7 @@ do_ping_pong_msg( )
        if ( polling ) {
            poll_count=0;
            LOGPRINTF("%d Polling for message receive event\n", getpid());
-           while (  dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY )
+           while (  dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )
                poll_count++;
        }
        else {
@@ -1292,15 +1342,16 @@ do_ping_pong_msg( )
                DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
                ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
                LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-               if ( evd != h_dto_evd )
+               if ( evd != h_dto_rcv_evd )
                {
-                   fprintf(stderr, "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
-                                       getpid());
+                   fprintf(stderr, 
+                          "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+                           getpid());
                    return( ret );
                }
            }
            /* use wait to dequeue */
-           ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+           ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
            if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
                                        getpid(),DT_RetToString(ret));
@@ -1511,6 +1562,7 @@ create_events(void)
     else {
         LOGPRINTF("%d cr_evd created %p\n", getpid(),h_cr_evd);
     }
+
     /* create conn EVD */
     ret = dat_evd_create( h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &h_conn_evd  );
     if(ret != DAT_SUCCESS) {
@@ -1522,19 +1574,34 @@ create_events(void)
         LOGPRINTF("%d con_evd created %p\n", getpid(),h_conn_evd);
     }
 
-    /* create dto EVD, with CNO if use_cno was set */
+    /* create dto SND EVD, with CNO if use_cno was set */
     ret = dat_evd_create( h_ia,
-                         (MSG_BUF_COUNT*2)+burst*2,
+                         MSG_BUF_COUNT+MAX_RDMA_RD+burst*2,
                          h_dto_cno,
                          DAT_EVD_DTO_FLAG,
-                         &h_dto_evd  );
+                         &h_dto_req_evd  );
     if(ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d Error dat_evd_create: %s\n",
+        fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
+                           getpid(),DT_RetToString(ret));
+        return (ret);
+    }
+    else {
+        LOGPRINTF("%d dto_req_evd created %p\n", getpid(), h_dto_req_evd );
+    }
+
+    /* create dto RCV EVD, with CNO if use_cno was set */
+    ret = dat_evd_create( h_ia,
+                         MSG_BUF_COUNT,
+                         h_dto_cno,
+                         DAT_EVD_DTO_FLAG,
+                         &h_dto_rcv_evd  );
+    if(ret != DAT_SUCCESS) {
+        fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
                            getpid(),DT_RetToString(ret));
         return (ret);
     }
     else {
-        LOGPRINTF("%d dto_evd created %p\n", getpid(), h_dto_evd );
+        LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(), h_dto_rcv_evd );
     }
 
     return DAT_SUCCESS;
@@ -1577,11 +1644,11 @@ destroy_events(void)
        }
     }
 
-    /* free dto EVD */
-    if ( h_dto_evd != DAT_HANDLE_NULL ) {
-        LOGPRINTF("%d Free dto EVD %p \n",getpid(),h_dto_evd);
+    /* free RCV dto EVD */
+    if ( h_dto_rcv_evd != DAT_HANDLE_NULL ) {
+        LOGPRINTF("%d Free RCV dto EVD %p \n",getpid(),h_dto_rcv_evd);
         start = get_time();
-       ret = dat_evd_free( h_dto_evd );
+       ret = dat_evd_free( h_dto_rcv_evd );
         stop = get_time();
        time.evdf += ((stop - start)*1.0e6);
        time.total += time.evdf;
@@ -1592,7 +1659,22 @@ destroy_events(void)
        }
        else {
            LOGPRINTF("%d Freed dto EVD\n",getpid());
-            h_dto_evd = DAT_HANDLE_NULL;
+            h_dto_rcv_evd = DAT_HANDLE_NULL;
+       }
+    }
+
+    /* free REQ dto EVD */
+    if ( h_dto_req_evd != DAT_HANDLE_NULL ) {
+        LOGPRINTF("%d Free REQ dto EVD %p \n",getpid(),h_dto_req_evd);
+        ret = dat_evd_free( h_dto_req_evd );
+        if(ret != DAT_SUCCESS) {
+           fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+                           getpid(), DT_RetToString(ret));
+           return (ret);
+       }
+       else {
+           LOGPRINTF("%d Freed dto EVD\n",getpid());
+            h_dto_req_evd = DAT_HANDLE_NULL;
        }
     }
 

diff --git a/test/dtest/makefile b/test/dtest/makefile
index 70722cf4feb69b2559dd3be69beb90965ae29384..e961fd7a0107623540d14a9ffddd2f880632d517 100644 (file)
--- a/test/dtest/makefile
+++ b/test/dtest/makefile
@@ -11,6 +11,6 @@ clean:
 
 dtest: ./dtest.c
        $(CC) $(CFLAGS) ./dtest.c -o dtest \
-       -DDAPL_PROVIDER='"IB1"' \
+       -DDAPL_PROVIDER='"OpenIB-ib0"' \
        -I $(DAT_INC) -L $(DAT_LIB) -ldat