dtest: enhancement to test, -D option for data check

With -D option, dtest will run pingpong rdma write test
with data validation. Changes pattern during iterations.
Aborts and reports location/pattern with any miscompare.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>
Signed-off-by: Amir Hanania <amir.hanania@intel.com>

diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
index 731ad60f4881cafd5a235a5b44ce77e7488c32c8..1541b7562eb8fac591d280def62247fb582f00d1 100755 (executable)
--- a/test/dtest/dtest.c
+++ b/test/dtest/dtest.c
@@ -197,7 +197,7 @@ struct dt_time {
 struct dt_time ts;
 
 /* defaults */
-static int all_data = 0;
+static int all_data_sizes = 0;
 static int increment = 0;
 static int failed = 0;
 static int uni_direction = 0;
@@ -228,6 +228,7 @@ static int burst_msg_index = 0;
 static int ucm = 0;
 static int rq_cnt, sq_cnt;
 static DAT_SOCK_ADDR6 remote;
+static int data_check = 0;
 
 /* forward prototypes */
 const char *DT_RetToStr(DAT_RETURN ret_value);
@@ -566,12 +567,12 @@ int main(int argc, char **argv)
        DAT_PROVIDER_ATTR pr_attr;
 
        /* parse arguments */
-       while ((c = getopt(argc, argv, "auwWtscvpb:d:B:h:P:S:i:")) != -1) {
+       while ((c = getopt(argc, argv, "UDauwWtscvpb:d:B:h:P:S:i:")) != -1) {
                switch (c) {
                case 'i':
                        increment = atoi(optarg);
                case 'a':
-                       all_data = 1;
+                       all_data_sizes = 1;
                        fflush(stdout);
                        break;
                case 'u':
@@ -582,6 +583,10 @@ int main(int argc, char **argv)
                        write_only = 1;
                        fflush(stdout);
                        break;
+               case 'D':
+                       data_check = 1;
+                       printf("%d Running DATA CHECK mode\n", getpid());
+                       /* fall through */
                case 'W':
                        write_only_pp = 1;
                        uni_direction = 1;
@@ -631,13 +636,15 @@ int main(int argc, char **argv)
                case 'S':
                        signal_rate = atoi(optarg);
                        break;
+               case 'U':
+                       /* fall through */
                default:
                        print_usage();
                        exit(-12);
                }
        }
 
-       if (all_data && !write_only_pp) {
+       if (all_data_sizes && !write_only_pp) {
                printf("\n\t -a option only valid with -W option\n\n");
                exit(-12);
        }
@@ -685,8 +692,9 @@ int main(int argc, char **argv)
 
        if (write_only_pp) {
                /* rdma write pingpong, default == 1 byte */
-               if (!all_data) {
-                       buf_len = 1;
+               if (!all_data_sizes) {
+                       if (!data_check)
+                               buf_len = 1;
                } else if (!increment) { /* power of 2 */
                        buf_len_p2 = 1;
                        i = 0;
@@ -890,7 +898,7 @@ int main(int argc, char **argv)
        if (write_only_pp) {
                int max, inc;
 
-               if (all_data) {
+               if (all_data_sizes) {
                        if (increment) {
                                i = 1;
                                inc = increment;
@@ -901,14 +909,44 @@ int main(int argc, char **argv)
                                max = buf_len_p2;
                        }
                } else {
-                       i = buf_len;
-                       max = buf_len;
-                       inc = buf_len;
+                       if (data_check) {
+                               i = buf_len;
+                               max = buf_len;
+                               inc = 1;
+                       }
+                       else
+                       {
+                               i = buf_len;
+                               max = buf_len;
+                               inc = buf_len;
+                       }
                }
-               printf("\n %d RDMA WRITE PINGPONG\n\n", getpid());
+               printf("\n %d RDMA WRITE PINGPONG %s\n\n", getpid(),
+                       data_check ? "with DATA CHECK":"");
+
                for (; i <= max; i++) {
-                       if (do_rdma_write_ping_pong(i, i*inc))
-                               break;
+                       if (all_data_sizes) {
+                               int l_len = (i*inc) ? (i*inc) : 1 << i;
+
+                               if ( l_len > 4 && do_rdma_write_ping_pong(i, l_len - 1)) {
+                                       fprintf(stderr, "%d Error do_rdma_write_ping_pong\n", getpid());
+                                       goto cleanup;
+                               }
+                       }
+
+                       if (do_rdma_write_ping_pong(i, i*inc)) {
+                               fprintf(stderr, "%d Error do_rdma_write_ping_pong\n", getpid());
+                               goto cleanup;
+                       }
+
+                       if (all_data_sizes) {
+                               int l_len = (i*inc) ? (i*inc) : 1 << i;
+
+                               if ( l_len > 1 && l_len < buf_len && do_rdma_write_ping_pong(i, l_len + 1)) {
+                                       fprintf(stderr, "%d Error do_rdma_write_ping_pong\n", getpid());
+                                       goto cleanup;
+                               }
+                       }
                }
        }
        else if (write_immed && write_only) {
@@ -1026,7 +1064,7 @@ complete:
        free(rbuf);
        free(sbuf);
 
-       if (ts.rtt && !all_data) {
+       if (ts.rtt && !all_data_sizes) {
                printf( "%d: %s PingPong: (%d x %d) Total %6.2lf us:"
                        " latency %3.2lf us, BW %4.2lf MB/s\n",
                        getpid(), write_only_pp ? "RDMA write":"Message",
@@ -1997,6 +2035,54 @@ acked:
        return (DAT_SUCCESS);
 }
 
+#define PAT_NUM 5
+unsigned char pat[PAT_NUM] = { 0, 0xff, 0x55, 0xaa, 0 };
+
+void set_pat(unsigned int len, unsigned int pat_num)
+{
+       if (len <= 1)
+               return;
+
+       if (pat_num >= PAT_NUM) {
+               printf("\n\tpat_num = %d. max valid number is %d.\n\n", pat_num, PAT_NUM - 1);
+               exit(1);
+       }
+
+       if (server) {
+               /* server */
+               if (pat_num == PAT_NUM - 1) {
+                       /* future: random data, add checksum */
+                       ;
+               } else {
+                       /* check first byte only for some speed */
+                       if ((unsigned char)rbuf[0] != (unsigned char)pat[pat_num]) {
+                               fprintf(stderr,"%d: ERR: message len is %d,"
+                                               " location 0. Rx 0x%x expected"
+                                               " 0x%x, pat %d\n",
+                                               getpid(), len, (unsigned char)rbuf[0],
+                                               (unsigned char)pat[pat_num], pat_num);
+                       }
+               }
+               memcpy(sbuf, rbuf, len - 1);
+
+       } else {
+               /* client */
+               int i;
+
+               if (pat_num == PAT_NUM - 1) { /* set random values */
+                       struct timeval tv;
+
+                       gettimeofday(&tv, NULL);
+                       srand((unsigned int)tv.tv_usec);
+                       for (i = 0; i < len - 1; i++)
+                               sbuf[i] = (unsigned char)rand();
+               } else {
+                       memset(sbuf, (unsigned char)pat[pat_num], len - 1);
+               }
+       }
+}
+
+
 /* always uni-direction */
 DAT_RETURN do_rdma_write_ping_pong(int p2, int bytes)
 {
@@ -2011,6 +2097,7 @@ DAT_RETURN do_rdma_write_ping_pong(int p2, int bytes)
        volatile char *tx_buf, *rx_buf;
        uint32_t rx_cnt = 0;
        uint32_t tx_cnt = 0;
+       unsigned char rx_idx = 0;
 
        len = bytes ? bytes : 1 << p2;
 
@@ -2035,6 +2122,34 @@ DAT_RETURN do_rdma_write_ping_pong(int p2, int bytes)
                if (rx_cnt < burst && !(!server && !tx_cnt)) {
                        rx_cnt++;
                        while (*rx_buf != (char)rx_cnt);
+                       rx_idx = (unsigned char)*rx_buf;
+
+                       if (data_check && !server && memcmp(sbuf, rbuf, len)) {
+                               int l=0, ll;
+                               fprintf(stderr, "%d: ERR: Tx data from server wrong\n", getpid());
+
+                               while (sbuf[l] == rbuf[l] && l < len)
+                                       l++;
+
+                               fprintf(stderr,"%d: len %d, 1st error at %d. Tx 0x%x Rx 0x%x\n",
+                                               getpid(), len, l, (unsigned char)sbuf[l],
+                                               (unsigned char)rbuf[l]);
+                               fprintf(stderr,"%d: rcnt %d (char = %d), tcnt %d, *rbuf %d\n",
+                                               getpid(), rx_cnt, (char)rx_cnt, tx_cnt,
+                                               (unsigned char)*rx_buf);
+                               fprintf(stderr, "Send:");
+
+                               for (ll=l; ll < len && ll < 1 + 64; ll++)
+                                       fprintf(stderr,"%02x", (unsigned char)sbuf[ll]);
+
+                               fprintf(stderr, "\nRecv:");
+
+                               for (ll=l; ll < len && ll < 1 + 64; ll++)
+                                       fprintf(stderr,"%02x", (unsigned char)rbuf[ll]);
+
+                               fprintf(stderr, "\n");
+                               return (DAT_ABORT);
+                       }
                }
 
                if (!((i+1) % signal_rate))
@@ -2045,6 +2160,9 @@ DAT_RETURN do_rdma_write_ping_pong(int p2, int bytes)
                if (tx_cnt == burst)
                        break;
 
+               if (data_check)
+                       set_pat(len, tx_cnt % PAT_NUM);
+
                *tx_buf = (char)++tx_cnt;
                cookie.as_64 = tx_cnt;
                ret = dat_ep_post_rdma_write(h_ep, MSG_IOV_COUNT,
@@ -2074,7 +2192,7 @@ DAT_RETURN do_rdma_write_ping_pong(int p2, int bytes)
        stop = get_time();
        ts.rtt = ((stop - start) * 1.0e6);
 
-       if ((unsigned char)*rx_buf != (unsigned char)rx_cnt) {
+       if (rx_idx != (unsigned char)rx_cnt) {
                printf( "%d %s RW pingpong: %p, last *buf %d != cnt %d\n",
                        getpid(), server ? "SERVER:" : "CLIENT:",
                        rx_buf, (unsigned char)*rx_buf,
@@ -2082,7 +2200,7 @@ DAT_RETURN do_rdma_write_ping_pong(int p2, int bytes)
                return (DAT_ABORT);
        }
 
-       if (all_data) {
+       if (all_data_sizes) {
                printf( "%d: RDMA write PingPong: (%d x %d) Total %6.2lf us:"
                        " latency %3.2lf us, BW %4.2lf MB/s\n",
                        getpid(), burst, len, ts.rtt, ts.rtt/burst/2,
@@ -2778,6 +2896,7 @@ void print_usage(void)
        printf("u: unidirectional bandwidth (default=bidirectional\n");
        printf("w: rdma write only, streaming\n");
        printf("W: rdma write only, ping pong\n");
+       printf("D: validate data in ping pong test\n");
        printf("t: performance times\n");
        printf("c: use cno\n");
        printf("a: all data sizes with rdma write pingpong \n");
@@ -2790,6 +2909,7 @@ void print_usage(void)
        printf("h: hostname/address of server, specified on client\n");
        printf("P: provider name (default = ofa-v2-mlx4_0-1u)\n");
        printf("S: signal_rate (default=10, completion every 10 iterations\n");
+       printf("U: print this Usage page\n");
        printf("\n");
 }