rsocket: Add option to specify size of inline data

Allow the user to override the default inline data size.
We still require a minimum size in order to transfer receive
buffer update message.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/include/rdma/rsocket.h b/include/rdma/rsocket.h
index 9fd99290682811a32cd33a520ed8095eb1e2ddd4..65feda96a0ff93e4371759d881b420d337fb2f60 100644 (file)
--- a/include/rdma/rsocket.h
+++ b/include/rdma/rsocket.h
@@ -75,7 +75,8 @@ int rgetsockname(int socket, struct sockaddr *addr, socklen_t *addrlen);
 #define SOL_RDMA 0x10000
 enum {
        RDMA_SQSIZE,
-       RDMA_RQSIZE
+       RDMA_RQSIZE,
+       RDMA_INLINE
 };
 
 int rsetsockopt(int socket, int level, int optname,

diff --git a/src/rsocket.c b/src/rsocket.c
index f94e48a56bf0517e9888df473278d6fd46788cd8..ef070a8557c593905b411db74b6a6768c6d8a3fa 100644 (file)
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -105,6 +105,7 @@ struct rs_sge {
        uint32_t length;
 };
 
+#define RS_MIN_INLINE    (sizeof(struct rs_sge))
 #define rs_host_is_net() (1 == htonl(1))
 #define RS_CONN_FLAG_NET 1
 
@@ -163,6 +164,7 @@ struct rsocket {
        uint16_t          sseq_no;
        uint16_t          sseq_comp;
        uint16_t          sq_size;
+       uint16_t          sq_inline;
 
        uint16_t          rq_size;
        uint16_t          rseq_no;
@@ -229,11 +231,13 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs)
        if (inherited_rs) {
                rs->sbuf_size = inherited_rs->sbuf_size;
                rs->rbuf_size = inherited_rs->rbuf_size;
+               rs->sq_inline = inherited_rs->sq_inline;
                rs->sq_size = inherited_rs->sq_size;
                rs->rq_size = inherited_rs->rq_size;
                rs->ctrl_avail = inherited_rs->ctrl_avail;
        } else {
                rs->sbuf_size = rs->rbuf_size = RS_BUF_SIZE;
+               rs->sq_inline = RS_INLINE;
                rs->sq_size = rs->rq_size = RS_QP_SIZE;
                rs->ctrl_avail = RS_QP_CTRL_SIZE;
        }
@@ -367,7 +371,7 @@ static int rs_create_ep(struct rsocket *rs)
        qp_attr.cap.max_recv_wr = rs->rq_size;
        qp_attr.cap.max_send_sge = 2;
        qp_attr.cap.max_recv_sge = 1;
-       qp_attr.cap.max_inline_data = RS_INLINE;
+       qp_attr.cap.max_inline_data = rs->sq_inline;
 
        ret = rdma_create_qp(rs->cm_id, NULL, &qp_attr);
        if (ret)
@@ -1156,7 +1160,7 @@ ssize_t rsend(int socket, const void *buf, size_t len, int flags)
                if (xfer_size > rs->target_sgl[rs->target_sge].length)
                        xfer_size = rs->target_sgl[rs->target_sge].length;
 
-               if (xfer_size <= RS_INLINE) {
+               if (xfer_size <= rs->sq_inline) {
                        sge.addr = (uintptr_t) buf;
                        sge.length = xfer_size;
                        sge.lkey = 0;
@@ -1277,7 +1281,7 @@ static ssize_t rsendv(int socket, const struct iovec *iov, int iovcnt, int flags
                        ret = rs_write_data(rs, rs_wrid(1, xfer_size),
                                            rs->ssgl, 1,
                                            rs_msg_set(RS_OP_DATA, xfer_size),
-                                           xfer_size <= RS_INLINE ? IBV_SEND_INLINE : 0);
+                                           xfer_size <= rs->sq_inline ? IBV_SEND_INLINE : 0);
                        if (xfer_size < rs_sbuf_left(rs))
                                rs->ssgl[0].addr += xfer_size;
                        else
@@ -1291,7 +1295,7 @@ static ssize_t rsendv(int socket, const struct iovec *iov, int iovcnt, int flags
                        ret = rs_write_data(rs, rs_wrid(1, xfer_size),
                                            rs->ssgl, 2,
                                            rs_msg_set(RS_OP_DATA, xfer_size),
-                                           xfer_size <= RS_INLINE ? IBV_SEND_INLINE : 0);
+                                           xfer_size <= rs->sq_inline ? IBV_SEND_INLINE : 0);
                        rs->ssgl[0].addr = (uintptr_t) rs->sbuf + rs->ssgl[1].length;
                }
                if (ret)
@@ -1730,6 +1734,11 @@ int rsetsockopt(int socket, int level, int optname,
                case RDMA_RQSIZE:
                        rs->rq_size = min((*(uint32_t *) optval), RS_QP_MAX_SIZE);
                        break;
+               case RDMA_INLINE:
+                       rs->sq_inline = min(*(uint32_t *) optval, RS_QP_MAX_SIZE);
+                       if (rs->sq_inline < RS_MIN_INLINE)
+                               rs->sq_inline = RS_MIN_INLINE;
+                       break;
                default:
                        break;
                }
@@ -1809,6 +1818,10 @@ int rgetsockopt(int socket, int level, int optname,
                        *((int *) optval) = rs->rq_size;
                        *optlen = sizeof(int);
                        break;
+               case RDMA_INLINE:
+                       *((int *) optval) = rs->sq_inline;
+                       *optlen = sizeof(int);
+                       break;
                default:
                        ret = ENOTSUP;
                        break;