rdma_client: handle IBV_SEND_INLINE correctly

Not all RDMA devices support IBV_SEND_INLINE.  At least some of those
that don't will ignore the flag passed to rdma_post_send and attempt to
send the command by using an sge entry instead.  Because we don't
register the send memory, this fails.  The proper way to deal with the
fact that IBV_SEND_INLINE is not guaranteed is to check the returned
value in our cap struct to see if we have support for inline data, and
if not, fall back to non-inline sends and to register the send memory
region.

Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/examples/rdma_client.c b/examples/rdma_client.c
index e0e176b4bdee3527caf30ee9e843ce842681ba95..f676b7023f0b8f762ecfcf21ae95a9f9d3cb04bb 100644 (file)
--- a/examples/rdma_client.c
+++ b/examples/rdma_client.c
@@ -39,7 +39,8 @@ static char *server = "127.0.0.1";
 static char *port = "7471";
 
 struct rdma_cm_id *id;
-struct ibv_mr *mr;
+struct ibv_mr *mr, *send_mr;
+int send_flags;
 uint8_t send_msg[16];
 uint8_t recv_msg[16];
 
@@ -65,6 +66,13 @@ static int run(void)
        attr.qp_context = id;
        attr.sq_sig_all = 1;
        ret = rdma_create_ep(&id, res, NULL, &attr);
+       // Check to see if we got inline data allowed or not
+       if (attr.cap.max_inline_data >= 16)
+               send_flags = IBV_SEND_INLINE;
+       else
+               printf("rdma_client: device doesn't support IBV_SEND_INLINE, "
+                      "using sge sends\n");
+
        if (ret) {
                perror("rdma_create_ep");
                goto out_free_addrinfo;
@@ -76,20 +84,28 @@ static int run(void)
                ret = -1;
                goto out_destroy_ep;
        }
+       if ((send_flags & IBV_SEND_INLINE) == 0) {
+               send_mr = rdma_reg_msgs(id, send_msg, 16);
+               if (!send_mr) {
+                       perror("rdma_reg_msgs for send_msg");
+                       ret = -1;
+                       goto out_dereg_recv;
+               }
+       }
 
        ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
        if (ret) {
                perror("rdma_post_recv");
-               goto out_dereg;
+               goto out_dereg_send;
        }
 
        ret = rdma_connect(id, NULL);
        if (ret) {
                perror("rdma_connect");
-               goto out_dereg;
+               goto out_dereg_send;
        }
 
-       ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+       ret = rdma_post_send(id, NULL, send_msg, 16, send_mr, send_flags);
        if (ret) {
                perror("rdma_post_send");
                goto out_disconnect;
@@ -109,7 +125,10 @@ static int run(void)
 
 out_disconnect:
        rdma_disconnect(id);
-out_dereg:
+out_dereg_send:
+       if ((send_flags & IBV_SEND_INLINE) == 0)
+               rdma_dereg_mr(send_mr);
+out_dereg_recv:
        rdma_dereg_mr(mr);
 out_destroy_ep:
        rdma_destroy_ep(id);