Get ready to receive before other side starts to send

Have server side of pingpong get ready to receive before client side
starts sending, to close a race condition.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c
index 0862d6ef5e1a9d7c65497b1f8bc5ffbd2a6924e4..85b4a4a72a262d920de63ee16570cefbb7a6a971 100644 (file)
--- a/examples/rc_pingpong.c
+++ b/examples/rc_pingpong.c
@@ -87,6 +87,56 @@ static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
        return attr.lid;
 }
 
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+                         struct pingpong_dest *dest)
+{
+       struct ibv_qp_attr attr = {
+               .qp_state               = IBV_QPS_RTR,
+               .path_mtu               = IBV_MTU_1024,
+               .dest_qp_num            = dest->qpn,
+               .rq_psn                 = dest->psn,
+               .max_dest_rd_atomic     = 1,
+               .min_rnr_timer          = 12,
+               .ah_attr                = {
+                       .is_global      = 0,
+                       .dlid           = dest->lid,
+                       .sl             = 0,
+                       .src_path_bits  = 0,
+                       .port_num       = port
+               }
+       };
+       if (ibv_modify_qp(ctx->qp, &attr,
+                         IBV_QP_STATE              |
+                         IBV_QP_AV                 |
+                         IBV_QP_PATH_MTU           |
+                         IBV_QP_DEST_QPN           |
+                         IBV_QP_RQ_PSN             |
+                         IBV_QP_MAX_DEST_RD_ATOMIC |
+                         IBV_QP_MIN_RNR_TIMER)) {
+               fprintf(stderr, "Failed to modify QP to RTR\n");
+               return 1;
+       }
+
+       attr.qp_state       = IBV_QPS_RTS;
+       attr.timeout        = 14;
+       attr.retry_cnt      = 7;
+       attr.rnr_retry      = 7;
+       attr.sq_psn         = my_psn;
+       attr.max_rd_atomic  = 1;
+       if (ibv_modify_qp(ctx->qp, &attr,
+                         IBV_QP_STATE              |
+                         IBV_QP_TIMEOUT            |
+                         IBV_QP_RETRY_CNT          |
+                         IBV_QP_RNR_RETRY          |
+                         IBV_QP_SQ_PSN             |
+                         IBV_QP_MAX_QP_RD_ATOMIC)) {
+               fprintf(stderr, "Failed to modify QP to RTS\n");
+               return 1;
+       }
+
+       return 0;
+}
+
 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
                                                 const struct pingpong_dest *my_dest)
 {
@@ -151,7 +201,9 @@ out:
        return rem_dest;
 }
 
-static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
+                                                int ib_port, int port,
+                                                const struct pingpong_dest *my_dest)
 {
        struct addrinfo *res, *t;
        struct addrinfo hints = {
@@ -215,6 +267,13 @@ static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong
 
        sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
 
+       if (pp_connect_ctx(ctx, ib_port, my_dest->psn, rem_dest)) {
+               fprintf(stderr, "Couldn't connect to remote QP\n");
+               free(rem_dest);
+               rem_dest = NULL;
+               goto out;
+       }
+
        sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
        if (write(connfd, msg, sizeof msg) != sizeof msg) {
                fprintf(stderr, "Couldn't send local address\n");
@@ -357,56 +416,6 @@ static int pp_post_send(struct pingpong_context *ctx)
        return ibv_post_send(ctx->qp, &wr, &bad_wr);
 }
 
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
-                         struct pingpong_dest *dest)
-{
-       struct ibv_qp_attr attr = {
-               .qp_state               = IBV_QPS_RTR,
-               .path_mtu               = IBV_MTU_1024,
-               .dest_qp_num            = dest->qpn,
-               .rq_psn                 = dest->psn,
-               .max_dest_rd_atomic     = 1,
-               .min_rnr_timer          = 12,
-               .ah_attr                = {
-                       .is_global      = 0,
-                       .dlid           = dest->lid,
-                       .sl             = 0,
-                       .src_path_bits  = 0,
-                       .port_num       = port
-               }
-       };
-       if (ibv_modify_qp(ctx->qp, &attr,
-                         IBV_QP_STATE              |
-                         IBV_QP_AV                 |
-                         IBV_QP_PATH_MTU           |
-                         IBV_QP_DEST_QPN           |
-                         IBV_QP_RQ_PSN             |
-                         IBV_QP_MAX_DEST_RD_ATOMIC |
-                         IBV_QP_MIN_RNR_TIMER)) {
-               fprintf(stderr, "Failed to modify QP to RTR\n");
-               return 1;
-       }
-
-       attr.qp_state       = IBV_QPS_RTS;
-       attr.timeout        = 14;
-       attr.retry_cnt      = 7;
-       attr.rnr_retry      = 7;
-       attr.sq_psn         = my_psn;
-       attr.max_rd_atomic  = 1;
-       if (ibv_modify_qp(ctx->qp, &attr,
-                         IBV_QP_STATE              |
-                         IBV_QP_TIMEOUT            |
-                         IBV_QP_RETRY_CNT          |
-                         IBV_QP_RNR_RETRY          |
-                         IBV_QP_SQ_PSN             |
-                         IBV_QP_MAX_QP_RD_ATOMIC)) {
-               fprintf(stderr, "Failed to modify QP to RTS\n");
-               return 1;
-       }
-
-       return 0;
-}
-
 static void usage(const char *argv0)
 {
        printf("Usage:\n");
@@ -556,7 +565,7 @@ int main(int argc, char *argv[])
        if (servername)
                rem_dest = pp_client_exch_dest(servername, port, &my_dest);
        else
-               rem_dest = pp_server_exch_dest(port, &my_dest);
+               rem_dest = pp_server_exch_dest(ctx, ib_port, port, &my_dest);
 
        if (!rem_dest)
                return 1;
@@ -564,8 +573,9 @@ int main(int argc, char *argv[])
        printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
               rem_dest->lid, rem_dest->qpn, rem_dest->psn);
 
-       if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
-               return 1;
+       if (servername)
+               if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
+                       return 1;
 
        if (use_event)
                if (ibv_req_notify_cq(ctx->cq, 0)) {

diff --git a/examples/uc_pingpong.c b/examples/uc_pingpong.c
index 8a9fb33e778f69b6fbc93f95690e3bd516b7cf2a..98617e9d855306f7119cc54d85b8ab6fce8060fd 100644 (file)
--- a/examples/uc_pingpong.c
+++ b/examples/uc_pingpong.c
@@ -87,6 +87,48 @@ static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
        return attr.lid;
 }
 
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+                         struct pingpong_dest *dest)
+{
+       struct ibv_qp_attr attr = {
+               .qp_state               = IBV_QPS_RTR,
+               .path_mtu               = IBV_MTU_1024,
+               .dest_qp_num            = dest->qpn,
+               .rq_psn                 = dest->psn,
+               .max_dest_rd_atomic     = 1,
+               .ah_attr                = {
+                       .is_global      = 0,
+                       .dlid           = dest->lid,
+                       .sl             = 0,
+                       .src_path_bits  = 0,
+                       .port_num       = port
+               }
+       };
+       if (ibv_modify_qp(ctx->qp, &attr,
+                         IBV_QP_STATE              |
+                         IBV_QP_AV                 |
+                         IBV_QP_PATH_MTU           |
+                         IBV_QP_DEST_QPN           |
+                         IBV_QP_RQ_PSN             |
+                         IBV_QP_MAX_DEST_RD_ATOMIC)) {
+               fprintf(stderr, "Failed to modify QP to RTR\n");
+               return 1;
+       }
+
+       attr.qp_state       = IBV_QPS_RTS;
+       attr.sq_psn         = my_psn;
+       attr.max_rd_atomic  = 1;
+       if (ibv_modify_qp(ctx->qp, &attr,
+                         IBV_QP_STATE              |
+                         IBV_QP_SQ_PSN             |
+                         IBV_QP_MAX_QP_RD_ATOMIC)) {
+               fprintf(stderr, "Failed to modify QP to RTS\n");
+               return 1;
+       }
+
+       return 0;
+}
+
 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
                                                 const struct pingpong_dest *my_dest)
 {
@@ -151,7 +193,9 @@ out:
        return rem_dest;
 }
 
-static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
+                                                int ib_port, int port,
+                                                const struct pingpong_dest *my_dest)
 {
        struct addrinfo *res, *t;
        struct addrinfo hints = {
@@ -215,6 +259,13 @@ static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong
 
        sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
 
+       if (pp_connect_ctx(ctx, ib_port, my_dest->psn, rem_dest)) {
+               fprintf(stderr, "Couldn't connect to remote QP\n");
+               free(rem_dest);
+               rem_dest = NULL;
+               goto out;
+       }
+
        sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
        if (write(connfd, msg, sizeof msg) != sizeof msg) {
                fprintf(stderr, "Couldn't send local address\n");
@@ -357,48 +408,6 @@ static int pp_post_send(struct pingpong_context *ctx)
        return ibv_post_send(ctx->qp, &wr, &bad_wr);
 }
 
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
-                         struct pingpong_dest *dest)
-{
-       struct ibv_qp_attr attr = {
-               .qp_state               = IBV_QPS_RTR,
-               .path_mtu               = IBV_MTU_1024,
-               .dest_qp_num            = dest->qpn,
-               .rq_psn                 = dest->psn,
-               .max_dest_rd_atomic     = 1,
-               .ah_attr                = {
-                       .is_global      = 0,
-                       .dlid           = dest->lid,
-                       .sl             = 0,
-                       .src_path_bits  = 0,
-                       .port_num       = port
-               }
-       };
-       if (ibv_modify_qp(ctx->qp, &attr,
-                         IBV_QP_STATE              |
-                         IBV_QP_AV                 |
-                         IBV_QP_PATH_MTU           |
-                         IBV_QP_DEST_QPN           |
-                         IBV_QP_RQ_PSN             |
-                         IBV_QP_MAX_DEST_RD_ATOMIC)) {
-               fprintf(stderr, "Failed to modify QP to RTR\n");
-               return 1;
-       }
-
-       attr.qp_state       = IBV_QPS_RTS;
-       attr.sq_psn         = my_psn;
-       attr.max_rd_atomic  = 1;
-       if (ibv_modify_qp(ctx->qp, &attr,
-                         IBV_QP_STATE              |
-                         IBV_QP_SQ_PSN             |
-                         IBV_QP_MAX_QP_RD_ATOMIC)) {
-               fprintf(stderr, "Failed to modify QP to RTS\n");
-               return 1;
-       }
-
-       return 0;
-}
-
 static void usage(const char *argv0)
 {
        printf("Usage:\n");
@@ -548,7 +557,7 @@ int main(int argc, char *argv[])
        if (servername)
                rem_dest = pp_client_exch_dest(servername, port, &my_dest);
        else
-               rem_dest = pp_server_exch_dest(port, &my_dest);
+               rem_dest = pp_server_exch_dest(ctx, ib_port, port, &my_dest);
 
        if (!rem_dest)
                return 1;
@@ -556,8 +565,9 @@ int main(int argc, char *argv[])
        printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
               rem_dest->lid, rem_dest->qpn, rem_dest->psn);
 
-       if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
-               return 1;
+       if (servername)
+               if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
+                       return 1;
 
        if (use_event)
                if (ibv_req_notify_cq(ctx->cq, 0)) {

diff --git a/examples/ud-pingpong.c b/examples/ud-pingpong.c
index 4936d309040ef3a86340b637438d0fd425efb4d6..0faf4295b6cddad1a37b5db9dd50b24262d6c8e0 100644 (file)
--- a/examples/ud-pingpong.c
+++ b/examples/ud-pingpong.c
@@ -78,7 +78,6 @@ struct pingpong_dest {
        int psn;
 };
 
-
 static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
 {
        struct ibv_port_attr attr;
@@ -89,6 +88,44 @@ static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
        return attr.lid;
 }
 
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+                         struct pingpong_dest *dest)
+{
+       struct ibv_qp_attr attr;
+       struct ibv_ah_attr ah_attr = {
+               .is_global     = 0,
+               .dlid          = dest->lid,
+               .sl            = 0,
+               .src_path_bits = 0,
+               .port_num      = port
+       };
+
+       attr.qp_state           = IBV_QPS_RTR;
+
+       if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
+               fprintf(stderr, "Failed to modify QP to RTR\n");
+               return 1;
+       }
+
+       attr.qp_state       = IBV_QPS_RTS;
+       attr.sq_psn         = my_psn;
+
+       if (ibv_modify_qp(ctx->qp, &attr,
+                         IBV_QP_STATE              |
+                         IBV_QP_SQ_PSN)) {
+               fprintf(stderr, "Failed to modify QP to RTS\n");
+               return 1;
+       }
+
+       ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
+       if (!ctx->ah) {
+               fprintf(stderr, "Failed to create AH\n");
+               return 1;
+       }
+
+       return 0;
+}
+
 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
                                                 const struct pingpong_dest *my_dest)
 {
@@ -153,7 +190,9 @@ out:
        return rem_dest;
 }
 
-static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
+                                                int ib_port, int port,
+                                                const struct pingpong_dest *my_dest)
 {
        struct addrinfo *res, *t;
        struct addrinfo hints = {
@@ -217,6 +256,13 @@ static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong
 
        sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
 
+       if (pp_connect_ctx(ctx, ib_port, my_dest->psn, rem_dest)) {
+               fprintf(stderr, "Couldn't connect to remote QP\n");
+               free(rem_dest);
+               rem_dest = NULL;
+               goto out;
+       }
+
        sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
        if (write(connfd, msg, sizeof msg) != sizeof msg) {
                fprintf(stderr, "Couldn't send local address\n");
@@ -366,44 +412,6 @@ static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
        return ibv_post_send(ctx->qp, &wr, &bad_wr);
 }
 
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
-                         struct pingpong_dest *dest)
-{
-       struct ibv_qp_attr attr;
-       struct ibv_ah_attr ah_attr = {
-               .is_global     = 0,
-               .dlid          = dest->lid,
-               .sl            = 0,
-               .src_path_bits = 0,
-               .port_num      = port
-       };
-
-       attr.qp_state           = IBV_QPS_RTR;
-
-       if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
-               fprintf(stderr, "Failed to modify QP to RTR\n");
-               return 1;
-       }
-
-       attr.qp_state       = IBV_QPS_RTS;
-       attr.sq_psn         = my_psn;
-
-       if (ibv_modify_qp(ctx->qp, &attr,
-                         IBV_QP_STATE              |
-                         IBV_QP_SQ_PSN)) {
-               fprintf(stderr, "Failed to modify QP to RTS\n");
-               return 1;
-       }
-
-       ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
-       if (!ctx->ah) {
-               fprintf(stderr, "Failed to create AH\n");
-               return 1;
-       }
-
-       return 0;
-}
-
 static void usage(const char *argv0)
 {
        printf("Usage:\n");
@@ -553,7 +561,7 @@ int main(int argc, char *argv[])
        if (servername)
                rem_dest = pp_client_exch_dest(servername, port, &my_dest);
        else
-               rem_dest = pp_server_exch_dest(port, &my_dest);
+               rem_dest = pp_server_exch_dest(ctx, ib_port, port, &my_dest);
 
        if (!rem_dest)
                return 1;
@@ -561,8 +569,9 @@ int main(int argc, char *argv[])
        printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
               rem_dest->lid, rem_dest->qpn, rem_dest->psn);
 
-       if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
-               return 1;
+       if (servername)
+               if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
+                       return 1;
 
        if (use_event)
                if (ibv_req_notify_cq(ctx->cq, 0)) {