Handle out-of-order completions in pingpong examples

Keep track of whether send and/or receive is pending in libibverbs
pingpong examples.  This avoids failures when the remote side receives
data and posts a send very quickly, and the local side completes the
receive before the previous send.  With the old code, this could
result in posting a send before the previous send completed, and
therefore overrun the send queue.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/ChangeLog b/ChangeLog
index 3f4716633b2461ec60faf28c13e7f8531a6db747..e90e01c2b8d24ef37c5b9d2aa037367593ef5af4 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2005-10-25  Roland Dreier  <roland@cisco.com>
+
+       * examples/rc_pingpong.c, examples/srq_pingpong.c,
+       examples/uc_pingpong.c, examples/ud_pingpong.c: Keep track of
+       whether send and/or receive is pending.  This avoids failures when
+       the remote side receives data and posts a send very quickly, and
+       the local side completes the receive before the previous send.
+       With the old code, this could result in posting a send before the
+       previous send completed, and therefore overrun the send queue.
+
 2005-10-23  Roland Dreier  <roland@cisco.com>
 
        * src/cmd.c (ibv_cmd_get_context_v2): Correct silly mistake in

diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c
index 8767514cbd75a47b5228b113c87e44463fe00ba5..947ae54003facaca1045a0d3461b439298030c04 100644 (file)
--- a/examples/rc_pingpong.c
+++ b/examples/rc_pingpong.c
@@ -70,6 +70,7 @@ struct pingpong_context {
        void                    *buf;
        int                      size;
        int                      rx_depth;
+       int                      pending;
 };
 
 struct pingpong_dest {
@@ -600,11 +601,15 @@ int main(int argc, char *argv[])
                        return 1;
                }
 
-       if (servername)
+       ctx->pending = PINGPONG_RECV_WRID;
+
+       if (servername) {
                if (pp_post_send(ctx)) {
                        fprintf(stderr, "Couldn't post send\n");
                        return 1;
                }
+               ctx->pending |= PINGPONG_SEND_WRID;
+       }
 
        if (gettimeofday(&start, NULL)) {
                perror("gettimeofday");
@@ -669,12 +674,6 @@ int main(int argc, char *argv[])
                                                }
                                        }
 
-                                       if (scnt < iters)
-                                               if (pp_post_send(ctx)) {
-                                                       fprintf(stderr, "Couldn't post send\n");
-                                                       return 1;
-                                               }
-
                                        ++rcnt;
                                        break;
 
@@ -683,6 +682,16 @@ int main(int argc, char *argv[])
                                                (int) wc[i].wr_id);
                                        return 1;
                                }
+
+                               ctx->pending &= ~(int) wc[i].wr_id;
+                               if (scnt < iters && !ctx->pending) {
+                                       if (pp_post_send(ctx)) {
+                                               fprintf(stderr, "Couldn't post send\n");
+                                               return 1;
+                                       }
+                                       ctx->pending = PINGPONG_RECV_WRID |
+                                                      PINGPONG_SEND_WRID;
+                               }
                        }
                }
        }

diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c
index c9108cae0298593e546a2786a697cbcb57413bdf..f8f7a3ec89ea14a02991396279fa6b97c4e3a88f 100644 (file)
--- a/examples/srq_pingpong.c
+++ b/examples/srq_pingpong.c
@@ -74,6 +74,7 @@ struct pingpong_context {
        int                      size;
        int                      num_qp;
        int                      rx_depth;
+       int                      pending[MAX_QP];
 };
 
 struct pingpong_dest {
@@ -403,7 +404,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
                        .recv_cq = ctx->cq,
                        .srq     = ctx->srq,
                        .cap     = {
-                               .max_send_wr  = 4,
+                               .max_send_wr  = 1,
                                .max_send_sge = 1,
                        },
                        .qp_type = IBV_QPT_RC
@@ -675,11 +676,16 @@ int main(int argc, char *argv[])
                }
 
        if (servername)
-               for (i = 0; i < num_qp; ++i)
+               for (i = 0; i < num_qp; ++i) {
                        if (pp_post_send(ctx, i)) {
                                fprintf(stderr, "Couldn't post send\n");
                                return 1;
                        }
+                       ctx->pending[i] = PINGPONG_SEND_WRID | PINGPONG_RECV_WRID;
+               }
+       else
+               for (i = 0; i < num_qp; ++i)
+                       ctx->pending[i] = PINGPONG_RECV_WRID;
 
        if (gettimeofday(&start, NULL)) {
                perror("gettimeofday");
@@ -710,7 +716,7 @@ int main(int argc, char *argv[])
 
                {
                        struct ibv_wc wc[2];
-                       int ne, j;
+                       int ne, qp_ind;
 
                        do {
                                ne = ibv_poll_cq(ctx->cq, 2, wc);
@@ -728,6 +734,13 @@ int main(int argc, char *argv[])
                                        return 1;
                                }
 
+                               qp_ind = find_qp(wc[i].qp_num, ctx, num_qp);
+                               if (qp_ind < 0) {
+                                       fprintf(stderr, "Couldn't find QPN %06x\n",
+                                               wc[i].qp_num);
+                                       return 1;
+                               }
+
                                switch ((int) wc[i].wr_id) {
                                case PINGPONG_SEND_WRID:
                                        ++scnt;
@@ -744,20 +757,6 @@ int main(int argc, char *argv[])
                                                }
                                        }
 
-                                       if (scnt < iters) {
-                                               j = find_qp(wc[i].qp_num, ctx, num_qp);
-                                               if (j < 0) {
-                                                       fprintf(stderr, "Couldn't find QPN %06x\n",
-                                                               wc[i].qp_num);
-                                                       return 1;
-                                               }
-
-                                               if (pp_post_send(ctx, j)) {
-                                                       fprintf(stderr, "Couldn't post send\n");
-                                                       return 1;
-                                               }
-                                       }
-
                                        ++rcnt;
                                        break;
 
@@ -766,6 +765,17 @@ int main(int argc, char *argv[])
                                                (int) wc[i].wr_id);
                                        return 1;
                                }
+
+                               ctx->pending[qp_ind] &= ~(int) wc[i].wr_id;
+                               if (scnt < iters && !ctx->pending[qp_ind]) {
+                                       if (pp_post_send(ctx, qp_ind)) {
+                                               fprintf(stderr, "Couldn't post send\n");
+                                               return 1;
+                                       }
+                                       ctx->pending[qp_ind] = PINGPONG_RECV_WRID |
+                                                              PINGPONG_SEND_WRID;
+                               }
+
                        }
                }
        }

diff --git a/examples/uc_pingpong.c b/examples/uc_pingpong.c
index aa68579173b86ee1d3cf755173120e7693da539b..5f25245b7797bb8efbdffca6d62698d0dcaf51c7 100644 (file)
--- a/examples/uc_pingpong.c
+++ b/examples/uc_pingpong.c
@@ -70,6 +70,7 @@ struct pingpong_context {
        void                    *buf;
        int                      size;
        int                      rx_depth;
+       int                      pending;
 };
 
 struct pingpong_dest {
@@ -592,11 +593,15 @@ int main(int argc, char *argv[])
                        return 1;
                }
 
-       if (servername)
+       ctx->pending = PINGPONG_RECV_WRID;
+
+       if (servername) {
                if (pp_post_send(ctx)) {
                        fprintf(stderr, "Couldn't post send\n");
                        return 1;
                }
+               ctx->pending |= PINGPONG_SEND_WRID;
+       }
 
        if (gettimeofday(&start, NULL)) {
                perror("gettimeofday");
@@ -661,12 +666,6 @@ int main(int argc, char *argv[])
                                                }
                                        }
 
-                                       if (scnt < iters)
-                                               if (pp_post_send(ctx)) {
-                                                       fprintf(stderr, "Couldn't post send\n");
-                                                       return 1;
-                                               }
-
                                        ++rcnt;
                                        break;
 
@@ -675,6 +674,16 @@ int main(int argc, char *argv[])
                                                (int) wc[i].wr_id);
                                        return 1;
                                }
+
+                               ctx->pending &= ~(int) wc[i].wr_id;
+                               if (scnt < iters && !ctx->pending) {
+                                       if (pp_post_send(ctx)) {
+                                               fprintf(stderr, "Couldn't post send\n");
+                                               return 1;
+                                       }
+                                       ctx->pending = PINGPONG_RECV_WRID |
+                                                      PINGPONG_SEND_WRID;
+                               }
                        }
                }
        }

diff --git a/examples/ud_pingpong.c b/examples/ud_pingpong.c
index c2d0c4cebc6078355ad3ab4f0696bc90e50bcef7..4fcf854f79a75a3eb7394a9d25057c995c393b6b 100644 (file)
--- a/examples/ud_pingpong.c
+++ b/examples/ud_pingpong.c
@@ -71,6 +71,7 @@ struct pingpong_context {
        void                    *buf;
        int                      size;
        int                      rx_depth;
+       int                      pending;
 };
 
 struct pingpong_dest {
@@ -596,11 +597,15 @@ int main(int argc, char *argv[])
                        return 1;
                }
 
-       if (servername)
+       ctx->pending = PINGPONG_RECV_WRID;
+
+       if (servername) {
                if (pp_post_send(ctx, rem_dest->qpn)) {
                        fprintf(stderr, "Couldn't post send\n");
                        return 1;
                }
+               ctx->pending |= PINGPONG_SEND_WRID;
+       }
 
        if (gettimeofday(&start, NULL)) {
                perror("gettimeofday");
@@ -665,12 +670,6 @@ int main(int argc, char *argv[])
                                                }
                                        }
 
-                                       if (scnt < iters)
-                                               if (pp_post_send(ctx, rem_dest->qpn)) {
-                                                       fprintf(stderr, "Couldn't post send\n");
-                                                       return 1;
-                                               }
-
                                        ++rcnt;
                                        break;
 
@@ -679,6 +678,16 @@ int main(int argc, char *argv[])
                                                (int) wc[i].wr_id);
                                        return 1;
                                }
+
+                               ctx->pending &= ~(int) wc[i].wr_id;
+                               if (scnt < iters && !ctx->pending) {
+                                       if (pp_post_send(ctx, rem_dest->qpn)) {
+                                               fprintf(stderr, "Couldn't post send\n");
+                                               return 1;
+                                       }
+                                       ctx->pending = PINGPONG_RECV_WRID |
+                                                      PINGPONG_SEND_WRID;
+                               }
                        }
                }
        }