Version: 1
-Previous: f1ed7bb1d97d263ef014cfb99bf9454f5e79acf6
+Previous: 590dcd2580933003e4fb7816ba23343bfcedf385
Head: 59cddb8c2cd143ffe205716eef276b9c502e1eb5
Applied:
cma-rm-pd: 2ffda7f2991395570b9e776ff5ae256ca9684771
rstream-fork: a1d261650896286989c5197199da7729b3d61fa5
init-getname: 7d988863b218d1b66e3739ec4b6f51acc72b2334
rs-ftp: 28e0744eb89227fbeded485fbad64010b9edf0f6
- mpi-finalize: 59cddb8c2cd143ffe205716eef276b9c502e1eb5
+ check-id: 59cddb8c2cd143ffe205716eef276b9c502e1eb5
Unapplied:
dbg: 0c269855776d3001e37da8c8afe283c20e1d6cd6
waitall-buggy: c49c6b56c55385774065f5aa2704078e6ae0ceb8
--- /dev/null
+Bottom: 2d7d36989522b9e010497256793038fb2cb5aa9e
+Top: b5df9cc04d6e7629ae937bfe9bbc3c7da5193ea0
+Author: Sean Hefty <sean.hefty@intel.com>
+Date: 2012-07-26 15:35:32 -0700
+
+rsockets: Use wr_id to determine completion type
+
+If a work request has completed in error, the completion type
+field is undefined. Use the wr_id to determine if the failed
+completion was a send or receive.
+
+This fixes an issue where MPI can hang during finalize. With
+both sides of a connection shutting down simultaneously, one
+side may complete quicker and delete its QP before the other
+side receives an acknowledgement to their disconnect message.
+Eventually, the disconnect message will time out, but because
+the completion type field is undefined, it may be processed
+as a failed receive, rather than a failed send. The end
+result is that the second side hangs waiting for the send to
+complete.
+
+This problem showed up more easily after commit
+2e5b0fc95964f74ea59dd725e849027faa0cd526, but existed beforehand.
+
+Signed-off-by: Sean Hefty <sean.hefty@intel.com>
+
+
+---
+
+diff --git a/src/rsocket.c b/src/rsocket.c
+index ed125b3..1e391b3 100644
+--- a/src/rsocket.c
++++ b/src/rsocket.c
+@@ -132,6 +132,8 @@ union rs_wr_id {
+ };
+ };
+
++#define RS_RECV_WR_ID (~((uint64_t) 0))
++
+ /*
+ * rsocket states are ordered as passive, connecting, connected, disconnected.
+ */
+@@ -418,6 +420,19 @@ err1:
+ return -1;
+ }
+
++static inline int
++rs_post_recv(struct rsocket *rs)
++{
++ struct ibv_recv_wr wr, *bad;
++
++ wr.wr_id = RS_RECV_WR_ID;
++ wr.next = NULL;
++ wr.sg_list = NULL;
++ wr.num_sge = 0;
++
++ return rdma_seterrno(ibv_post_recv(rs->cm_id->qp, &wr, &bad));
++}
++
+ static int rs_create_ep(struct rsocket *rs)
+ {
+ struct ibv_qp_init_attr qp_attr;
+@@ -449,7 +464,7 @@ static int rs_create_ep(struct rsocket *rs)
+ return ret;
+
+ for (i = 0; i < rs->rq_size; i++) {
+- ret = rdma_post_recvv(rs->cm_id, NULL, NULL, 0);
++ ret = rs_post_recv(rs);
+ if (ret)
+ return ret;
+ }
+@@ -881,7 +896,7 @@ static int rs_poll_cq(struct rsocket *rs)
+ int ret, rcnt = 0;
+
+ while ((ret = ibv_poll_cq(rs->cm_id->recv_cq, 1, &wc)) > 0) {
+- if (wc.opcode == IBV_WC_RECV_RDMA_WITH_IMM) {
++ if (wc.wr_id == RS_RECV_WR_ID) {
+ if (wc.status != IBV_WC_SUCCESS)
+ continue;
+ rcnt++;
+@@ -923,7 +938,7 @@ static int rs_poll_cq(struct rsocket *rs)
+
+ if (rs->state & rs_connected) {
+ while (!ret && rcnt--)
+- ret = rdma_post_recvv(rs->cm_id, NULL, NULL, 0);
++ ret = rs_post_recv(rs);
+
+ if (ret) {
+ rs->state = rs_error;
+++ /dev/null
-Bottom: 2d7d36989522b9e010497256793038fb2cb5aa9e
-Top: b5df9cc04d6e7629ae937bfe9bbc3c7da5193ea0
-Author: Sean Hefty <sean.hefty@intel.com>
-Date: 2012-07-26 15:35:32 -0700
-
-rsockets: Use wr_id to determine completion type
-
-If a work request has completed in error, the completion type
-field is undefined. Use the wr_id to determine if the failed
-completion was a send or receive.
-
-This fixes an issue where MPI can hang during finalize. With
-both sides of a connection shutting down simultaneously, one
-side may complete quicker and delete its QP before the other
-side receives an acknowledgement to their disconnect message.
-Eventually, the disconnect message will time out, but because
-the completion type field is undefined, it may be processed
-as a failed receive, rather than a failed send. The end
-result is that the second side hangs waiting for the send to
-complete.
-
-This problem showed up more easily after commit
-2e5b0fc95964f74ea59dd725e849027faa0cd526, but existed beforehand.
-
-Signed-off-by: Sean Hefty <sean.hefty@intel.com>
-
-
----
-
-diff --git a/src/rsocket.c b/src/rsocket.c
-index ed125b3..1e391b3 100644
---- a/src/rsocket.c
-+++ b/src/rsocket.c
-@@ -132,6 +132,8 @@ union rs_wr_id {
- };
- };
-
-+#define RS_RECV_WR_ID (~((uint64_t) 0))
-+
- /*
- * rsocket states are ordered as passive, connecting, connected, disconnected.
- */
-@@ -418,6 +420,19 @@ err1:
- return -1;
- }
-
-+static inline int
-+rs_post_recv(struct rsocket *rs)
-+{
-+ struct ibv_recv_wr wr, *bad;
-+
-+ wr.wr_id = RS_RECV_WR_ID;
-+ wr.next = NULL;
-+ wr.sg_list = NULL;
-+ wr.num_sge = 0;
-+
-+ return rdma_seterrno(ibv_post_recv(rs->cm_id->qp, &wr, &bad));
-+}
-+
- static int rs_create_ep(struct rsocket *rs)
- {
- struct ibv_qp_init_attr qp_attr;
-@@ -449,7 +464,7 @@ static int rs_create_ep(struct rsocket *rs)
- return ret;
-
- for (i = 0; i < rs->rq_size; i++) {
-- ret = rdma_post_recvv(rs->cm_id, NULL, NULL, 0);
-+ ret = rs_post_recv(rs);
- if (ret)
- return ret;
- }
-@@ -881,7 +896,7 @@ static int rs_poll_cq(struct rsocket *rs)
- int ret, rcnt = 0;
-
- while ((ret = ibv_poll_cq(rs->cm_id->recv_cq, 1, &wc)) > 0) {
-- if (wc.opcode == IBV_WC_RECV_RDMA_WITH_IMM) {
-+ if (wc.wr_id == RS_RECV_WR_ID) {
- if (wc.status != IBV_WC_SUCCESS)
- continue;
- rcnt++;
-@@ -923,7 +938,7 @@ static int rs_poll_cq(struct rsocket *rs)
-
- if (rs->state & rs_connected) {
- while (!ret && rcnt--)
-- ret = rdma_post_recvv(rs->cm_id, NULL, NULL, 0);
-+ ret = rs_post_recv(rs);
-
- if (ret) {
- rs->state = rs_error;