From: Sean Hefty Date: Sun, 20 May 2012 23:20:49 +0000 (-0700) Subject: refresh (create temporary patch) X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=e56faeca7ea3fbb0a41bdbb2d8191436e1728b22;p=~shefty%2Flibrdmacm.git refresh (create temporary patch) --- diff --git a/meta b/meta index 5de734c5..080997ff 100644 --- a/meta +++ b/meta @@ -1,8 +1,9 @@ Version: 1 -Previous: 7b2a62ef9a53bb66039b2a48a3a849a838a50626 -Head: a75aec6078074006c6ab9242a74901adf11a0e73 +Previous: e14fd986edf6993777469bff457dfe7933924af9 +Head: 2ec76378b3757b05ac1cba32320d0122ca344b7c Applied: rs-qpsize: a75aec6078074006c6ab9242a74901adf11a0e73 + refresh-temp: 2ec76378b3757b05ac1cba32320d0122ca344b7c Unapplied: rs-opts: 7c8fbc95903cf8f61422d7f2c58f83ba4f0b8ac4 rs-opt-qpsize: 6b732ff34d98cf8745e2f5d2ff326cef4a3b05a7 diff --git a/patches/refresh-temp b/patches/refresh-temp new file mode 100644 index 00000000..73277650 --- /dev/null +++ b/patches/refresh-temp @@ -0,0 +1,232 @@ +Bottom: 6bbc2f526b97dfafc16e7ec34dc8e4e1ce587f40 +Top: b37d006a6372969ddf786ae8c3abe0509cfaef04 +Author: Sean Hefty +Date: 2012-05-20 16:20:49 -0700 + +Refresh of rs-qpsize + +--- + +diff --git a/src/cma.h b/src/cma.h +index 7703fe8..669b681 100644 +--- a/src/cma.h ++++ b/src/cma.h +@@ -67,7 +67,7 @@ static inline uint64_t htonll(uint64_t x) { return x; } + static inline uint64_t ntohll(uint64_t x) { return x; } + #endif + +-#define min(a, b) (a < b ? a : b) ++#define min(a, b) ((a) < (b) ? a : b) + + /* + * Fast synchronization for low contention locking. +diff --git a/src/rsocket.c b/src/rsocket.c +index de0ec3e..495c2e3 100644 +--- a/src/rsocket.c ++++ b/src/rsocket.c +@@ -55,6 +55,8 @@ + #define RS_OLAP_START_SIZE 2048 + #define RS_MAX_TRANSFER 65536 + #define RS_QP_SIZE 512 ++#define RS_QP_MAX_SIZE 0xFFFE ++#define RS_QP_MIN_SIZE 8 + #define RS_QP_CTRL_SIZE 4 + #define RS_CONN_RETRIES 6 + #define RS_SGL_SIZE 2 +@@ -160,7 +162,9 @@ struct rsocket { + int sbuf_bytes_avail; + uint16_t sseq_no; + uint16_t sseq_comp; ++ uint16_t sq_size; + ++ uint16_t rq_size; + uint16_t rseq_no; + uint16_t rseq_comp; + int rbuf_bytes_avail; +@@ -168,7 +172,7 @@ struct rsocket { + int rbuf_offset; + int rmsg_head; + int rmsg_tail; +- struct rs_msg rmsg[RS_QP_SIZE + 1]; ++ struct rs_msg *rmsg; + + int remote_sge; + struct rs_sge remote_sgl; +@@ -222,8 +226,17 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs) + return NULL; + + rs->index = -1; +- rs->sbuf_size = inherited_rs ? inherited_rs->sbuf_size : RS_BUF_SIZE; +- rs->rbuf_size = inherited_rs ? inherited_rs->rbuf_size : RS_BUF_SIZE; ++ if (inherited_rs) { ++ rs->sbuf_size = inherited_rs->sbuf_size; ++ rs->rbuf_size = inherited_rs->rbuf_size; ++ rs->sq_size = inherited_rs->sq_size; ++ rs->rq_size = inherited_rs->rq_size; ++ rs->ctrl_avail = inherited_rs->ctrl_avail; ++ } else { ++ rs->sbuf_size = rs->rbuf_size = RS_BUF_SIZE; ++ rs->sq_size = rs->rq_size = RS_QP_SIZE; ++ rs->ctrl_avail = RS_QP_CTRL_SIZE; ++ } + fastlock_init(&rs->slock); + fastlock_init(&rs->rlock); + fastlock_init(&rs->cq_lock); +@@ -244,8 +257,27 @@ static int rs_set_nonblocking(struct rsocket *rs, long arg) + return ret; + } + ++static void rs_set_qp_size(struct rsocket *rs) ++{ ++ struct ibv_device_attr attr; ++ uint16_t max_size; ++ ++ attr.max_qp_wr = RS_QP_SIZE; ++ ibv_query_device(rs->cm_id->verbs, &attr); ++ max_size = min(attr.max_qp_wr, RS_QP_MAX_SIZE); ++ ++ if (rs->sq_size > max_size) ++ rs->sq_size = max_size; ++ if (rs->rq_size > max_size) ++ rs->rq_size = max_size; ++} ++ + static int rs_init_bufs(struct rsocket *rs) + { ++ rs->rmsg = calloc(rs->rq_size + 1, sizeof(*rs->rmsg)); ++ if (!rs->rmsg) ++ return -1; ++ + rs->sbuf = calloc(rs->sbuf_size, sizeof(*rs->sbuf)); + if (!rs->sbuf) + return -1; +@@ -273,9 +305,8 @@ static int rs_init_bufs(struct rsocket *rs) + + rs->rbuf_free_offset = rs->rbuf_size >> 1; + rs->rbuf_bytes_avail = rs->rbuf_size >> 1; +- rs->ctrl_avail = RS_QP_CTRL_SIZE; +- rs->sqe_avail = RS_QP_SIZE - rs->ctrl_avail; +- rs->rseq_comp = RS_QP_SIZE >> 1; ++ rs->sqe_avail = rs->sq_size - rs->ctrl_avail; ++ rs->rseq_comp = rs->rq_size >> 1; + return 0; + } + +@@ -285,7 +316,7 @@ static int rs_create_cq(struct rsocket *rs) + if (!rs->cm_id->recv_cq_channel) + return -1; + +- rs->cm_id->recv_cq = ibv_create_cq(rs->cm_id->verbs, RS_QP_SIZE * 2, ++ rs->cm_id->recv_cq = ibv_create_cq(rs->cm_id->verbs, rs->sq_size + rs->rq_size, + rs->cm_id, rs->cm_id->recv_cq_channel, 0); + if (!rs->cm_id->recv_cq) + goto err1; +@@ -313,6 +344,7 @@ static int rs_create_ep(struct rsocket *rs) + struct ibv_qp_init_attr qp_attr; + int i, ret; + ++ rs_check_qp_size(rs); + ret = rs_init_bufs(rs); + if (ret) + return ret; +@@ -327,8 +359,8 @@ static int rs_create_ep(struct rsocket *rs) + qp_attr.recv_cq = rs->cm_id->recv_cq; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.sq_sig_all = 1; +- qp_attr.cap.max_send_wr = RS_QP_SIZE; +- qp_attr.cap.max_recv_wr = RS_QP_SIZE; ++ qp_attr.cap.max_send_wr = rs->sq_size; ++ qp_attr.cap.max_recv_wr = rs->rq_size; + qp_attr.cap.max_send_sge = 2; + qp_attr.cap.max_recv_sge = 1; + qp_attr.cap.max_inline_data = RS_INLINE; +@@ -337,7 +369,7 @@ static int rs_create_ep(struct rsocket *rs) + if (ret) + return ret; + +- for (i = 0; i < RS_QP_SIZE; i++) { ++ for (i = 0; i < rs->rq_size; i++) { + ret = rdma_post_recvv(rs->cm_id, NULL, NULL, 0); + if (ret) + return ret; +@@ -350,6 +382,9 @@ static void rs_free(struct rsocket *rs) + if (rs->index >= 0) + rs_remove(rs); + ++ if (rs->rmsg) ++ free(rs->rmsg); ++ + if (rs->sbuf) { + if (rs->smr) + rdma_dereg_mr(rs->smr); +@@ -386,7 +421,7 @@ static void rs_set_conn_data(struct rsocket *rs, struct rdma_conn_param *param, + { + conn->version = 1; + conn->flags = rs_host_is_net() ? RS_CONN_FLAG_NET : 0; +- conn->credits = htons(RS_QP_SIZE); ++ conn->credits = htons(rs->rq_size); + conn->reserved2 = 0; + + conn->target_sgl.addr = htonll((uintptr_t) rs->target_sgl); +@@ -706,7 +741,7 @@ static void rs_send_credits(struct rsocket *rs) + struct rs_sge sge; + + rs->ctrl_avail--; +- rs->rseq_comp = rs->rseq_no + (RS_QP_SIZE >> 1); ++ rs->rseq_comp = rs->rseq_no + (rs->rq_size >> 1); + if (rs->rbuf_bytes_avail >= (rs->rbuf_size >> 1)) { + if (!(rs->opts & RS_OPT_SWAP_SGL)) { + sge.addr = (uintptr_t) &rs->rbuf[rs->rbuf_free_offset]; +@@ -723,7 +758,7 @@ static void rs_send_credits(struct rsocket *rs) + ibsge.length = sizeof(sge); + + rs_post_write(rs, 0, &ibsge, 1, +- rs_msg_set(RS_OP_SGL, rs->rseq_no + RS_QP_SIZE), ++ rs_msg_set(RS_OP_SGL, rs->rseq_no + rs->rq_size), + IBV_SEND_INLINE, + rs->remote_sgl.addr + + rs->remote_sge * sizeof(struct rs_sge), +@@ -737,7 +772,7 @@ static void rs_send_credits(struct rsocket *rs) + rs->remote_sge = 0; + } else { + rs_post_write(rs, 0, NULL, 0, +- rs_msg_set(RS_OP_SGL, rs->rseq_no + RS_QP_SIZE), 0, 0, 0); ++ rs_msg_set(RS_OP_SGL, rs->rseq_no + rs->rq_size), 0, 0, 0); + } + } + +@@ -781,7 +816,7 @@ static int rs_poll_cq(struct rsocket *rs) + default: + rs->rmsg[rs->rmsg_tail].op = rs_msg_op(imm_data); + rs->rmsg[rs->rmsg_tail].data = rs_msg_data(imm_data); +- if (++rs->rmsg_tail == RS_QP_SIZE + 1) ++ if (++rs->rmsg_tail == rs->rq_size + 1) + rs->rmsg_tail = 0; + break; + } +@@ -923,7 +958,7 @@ static int rs_have_rdata(struct rsocket *rs) + + static int rs_all_sends_done(struct rsocket *rs) + { +- return (rs->sqe_avail + rs->ctrl_avail) == RS_QP_SIZE; ++ return (rs->sqe_avail + rs->ctrl_avail) == rs->sq_size; + } + + static ssize_t rs_peek(struct rsocket *rs, void *buf, size_t len) +@@ -940,7 +975,7 @@ static ssize_t rs_peek(struct rsocket *rs, void *buf, size_t len) + rsize = left; + } else { + rsize = rs->rmsg[rmsg_head].data; +- if (++rmsg_head == RS_QP_SIZE + 1) ++ if (++rmsg_head == rs->rq_size + 1) + rmsg_head = 0; + } + +@@ -1001,7 +1036,7 @@ ssize_t rrecv(int socket, void *buf, size_t len, int flags) + } else { + rs->rseq_no++; + rsize = rs->rmsg[rs->rmsg_head].data; +- if (++rs->rmsg_head == RS_QP_SIZE + 1) ++ if (++rs->rmsg_head == rs->rq_size + 1) + rs->rmsg_head = 0; + }