From 6a9e4b7aca9c73d8fa23c183bc25efe6b84355d2 Mon Sep 17 00:00:00 2001
From: Sean Hefty <sean.hefty@intel.com>
Date: Thu, 17 Apr 2014 18:15:38 -0700
Subject: [PATCH] Refresh of inline0

---
 src/rsocket.c | 117 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 76 insertions(+), 41 deletions(-)

diff --git a/src/rsocket.c b/src/rsocket.c
index 07cd8d35..86eedd28 100644
--- a/src/rsocket.c
+++ b/src/rsocket.c
@@ -1614,49 +1614,75 @@ static void *rs_get_ctrl_buf(struct rsocket *rs)
 		RS_MAX_CTRL_MSG * (rs->ctrl_seqno & (RS_QP_CTRL_SIZE - 1));
 }
 
-static int rs_post_send_msg(struct rsocket *rs, uint64_t wr_id, uint32_t msg)
+//static int rs_post_send_msg(struct rsocket *rs, uint64_t wr_id, uint32_t msg)
+//{
+//	struct ibv_send_wr wr, *bad;
+//	struct ibv_sge sge;
+//	void *ctrl_buf;
+//
+//	wr.wr_id = wr_id;
+//	wr.next = NULL;
+//	if (rs->sq_inline < sizeof msg) {
+//		ctrl_buf = rs_get_ctrl_buf(rs);
+//		memcpy(ctrl_buf, &msg, sizeof msg);
+//		sge.addr = (uintptr_t) ctrl_buf;
+//		sge.lkey = rs->smr->lkey;
+//		sge.length = sizeof msg;
+//		wr.send_flags = 0;
+//	} else {
+//		sge.addr = (uintptr_t) &msg;
+//		sge.lkey = 0;
+//		sge.length = sizeof msg;
+//		wr.send_flags = IBV_SEND_INLINE;
+//	}
+//	wr.sg_list = &sge;
+//	wr.num_sge = 1;
+//	wr.opcode = IBV_WR_SEND;
+//
+//	return rdma_seterrno(ibv_post_send(rs->cm_id->qp, &wr, &bad));
+//}
+//
+//static int rs_post_msg(struct rsocket *rs, uint32_t msg)
+//{
+//	struct ibv_send_wr wr, *bad;
+//	struct ibv_sge sge;
+//
+//	if (rs->opts & RS_OPT_MSG_SEND)
+//		return rs_post_send_msg(rs, rs_send_wr_id(msg), msg);
+//
+//	wr.wr_id = rs_send_wr_id(msg);
+//	wr.next = NULL;
+//	wr.sg_list = NULL;
+//	wr.num_sge = 0;
+//	wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
+//	wr.send_flags = 0;
+//	wr.imm_data = htonl(msg);
+//
+//	return rdma_seterrno(ibv_post_send(rs->cm_id->qp, &wr, &bad));
+//}
+
+static int rs_post_msg(struct rsocket *rs, uint32_t msg)
 {
 	struct ibv_send_wr wr, *bad;
 	struct ibv_sge sge;
-	void *ctrl_buf;
 
-	wr.wr_id = wr_id;
+	wr.wr_id = rs_send_wr_id(msg);
 	wr.next = NULL;
-	if (rs->sq_inline < sizeof msg) {
-		ctrl_buf = rs_get_ctrl_buf(rs);
-		memcpy(ctrl_buf, &msg, sizeof msg);
-		sge.addr = (uintptr_t) ctrl_buf;
-		sge.lkey = rs->smr->lkey;
-		sge.length = sizeof msg;
+	if (!(rs->opts & RS_OPT_MSG_SEND)) {
+		wr.sg_list = NULL;
+		wr.num_sge = 0;
+		wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
 		wr.send_flags = 0;
+		wr.imm_data = htonl(msg);
 	} else {
 		sge.addr = (uintptr_t) &msg;
 		sge.lkey = 0;
 		sge.length = sizeof msg;
+		wr.sg_list = &sge;
+		wr.num_sge = 1;
+		wr.opcode = IBV_WR_SEND;
 		wr.send_flags = IBV_SEND_INLINE;
 	}
-	wr.sg_list = &sge;
-	wr.num_sge = 1;
-	wr.opcode = IBV_WR_SEND;
-
-	return rdma_seterrno(ibv_post_send(rs->cm_id->qp, &wr, &bad));
-}
-
-static int rs_post_msg(struct rsocket *rs, uint32_t msg)
-{
-	struct ibv_send_wr wr, *bad;
-	struct ibv_sge sge;
-
-	if (rs->opts & RS_OPT_MSG_SEND)
-		return rs_post_send_msg(rs, rs_send_wr_id(msg), msg);
-
-	wr.wr_id = rs_send_wr_id(msg);
-	wr.next = NULL;
-	wr.sg_list = NULL;
-	wr.num_sge = 0;
-	wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
-	wr.send_flags = 0;
-	wr.imm_data = htonl(msg);
 
 	return rdma_seterrno(ibv_post_send(rs->cm_id->qp, &wr, &bad));
 }
@@ -1680,9 +1706,10 @@ static int rs_post_write(struct rsocket *rs,
 	return rdma_seterrno(ibv_post_send(rs->cm_id->qp, &wr, &bad));
 }
 
-static int
-rs_post_write_with_msg(struct rsocket *rs, struct ibv_sge *sgl, int nsge,
-		       uint32_t msg, int flags, uint64_t addr, uint32_t rkey)
+static int rs_post_write_msg(struct rsocket *rs,
+			 struct ibv_sge *sgl, int nsge,
+			 uint32_t msg, int flags,
+			 uint64_t addr, uint32_t rkey)
 {
 	struct ibv_send_wr wr, *bad;
 	struct ibv_sge sge;
@@ -1703,9 +1730,17 @@ rs_post_write_with_msg(struct rsocket *rs, struct ibv_sge *sgl, int nsge,
 	} else {
 		ret = rs_post_write(rs, sgl, nsge, msg, flags, addr, rkey);
 		if (!ret) {
-			ret = rs_post_send_msg(rs,
-				rs_send_wr_id(rs_msg_set(rs_msg_op(msg), 0)) |
-				RS_WR_ID_FLAG_MSG_SEND, msg);
+			wr.wr_id = rs_send_wr_id(rs_msg_set(rs_msg_op(msg), 0)) |
+				   RS_WR_ID_FLAG_MSG_SEND;
+			sge.addr = (uintptr_t) &msg;
+			sge.lkey = 0;
+			sge.length = sizeof msg;
+			wr.sg_list = &sge;
+			wr.num_sge = 1;
+			wr.opcode = IBV_WR_SEND;
+			wr.send_flags = IBV_SEND_INLINE;
+
+			ret = rdma_seterrno(ibv_post_send(rs->cm_id->qp, &wr, &bad));
 		}
 		return ret;
 	}
@@ -1757,8 +1792,8 @@ static int rs_write_data(struct rsocket *rs,
 			rs->target_sge = 0;
 	}
 
-	return rs_post_write_with_msg(rs, sgl, nsge, rs_msg_set(RS_OP_DATA, length),
-				      flags, addr, rkey);
+	return rs_post_write_msg(rs, sgl, nsge, rs_msg_set(RS_OP_DATA, length),
+				 flags, addr, rkey);
 }
 
 static int rs_write_direct(struct rsocket *rs, struct rs_iomap *iom, uint64_t offset,
@@ -1786,7 +1821,7 @@ static int rs_write_iomap(struct rsocket *rs, struct rs_iomap_mr *iomr,
 	rs->sbuf_bytes_avail -= sizeof(struct rs_iomap);
 
 	addr = rs->remote_iomap.addr + iomr->index * sizeof(struct rs_iomap);
-	return rs_post_write_with_msg(rs, sgl, nsge,
+	return rs_post_write_msg(rs, sgl, nsge,
 		rs_msg_set(RS_OP_IOMAP_SGL, iomr->index), flags, addr,
 		rs->remote_iomap.key);
 }
@@ -1822,7 +1857,7 @@ static void rs_send_credits(struct rsocket *rs)
 		ibsge.lkey = 0;
 		ibsge.length = sizeof(sge);
 
-		rs_post_write_with_msg(rs, &ibsge, 1,
+		rs_post_write_msg(rs, &ibsge, 1,
 			rs_msg_set(RS_OP_SGL, rs->rseq_no + rs->rq_size),
 			IBV_SEND_INLINE,
 			rs->remote_sgl.addr + rs->remote_sge * sizeof(struct rs_sge),
-- 
2.41.0