mpxyd: m_req_event assert during large io streams, HST to MIC

When proxy-in (PI) WR queue is full and client is blocked on
new WR entries, the WR completion processing can
incorrectly reference a PI WR field after the client is
given remote access.

m_qp = (struct mcm_qp *)m_wr_rx->context;
assert(m_qp);

After data is forwarded to the appropriate MIC, the proxy
service will send a RW_imm WC message. This releases
the m_wr_rx entry for re-use by remote mcm provider client.
At the same time, the proxy can be processing the RW_imm
completion and incorrectly use the wr_rx->context field for
m_qp reference. Change the proxy_in event processing code
to avoid dependencies on any wr_rx content.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/svc/mpxy_in.c b/dapl/svc/mpxy_in.c
index 5c82bd8774d2a2187da05ed479af2504e784523f..9f38b298244daa622ec6cdbfd2efaec35cc04bc1 100644 (file)
--- a/dapl/svc/mpxy_in.c
+++ b/dapl/svc/mpxy_in.c
@@ -553,7 +553,6 @@ static int m_pi_send_wc(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_rx, int status
                        wr_rx->wr.wr.rdma.remote_addr, wr_rx->wr.wr.rdma.rkey);
                return ret;
        }
-       m_qp->pi_rw_cnt++;
 
        mlog(4," WC_rem hd %d tl %d, m_qp %p qpn 0x%x, m_cm %p\n",
                m_qp->wc_hd_rem, m_qp->wc_tl_rem, m_qp, ib_qp->qp_num, m_qp->cm);
@@ -801,15 +800,6 @@ void m_pi_req_event(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_rx, struct ibv_wc
                mpxy_unlock(&m_qp->rxlock);
                write(m_qp->smd->md->mc->rx_pipe[1], "w", sizeof "w"); /* signal rx_thread */
                sched_yield();
-       } else {
-               mlog(4, " WR_rx[%d] %p flgs %x %s complete - WR tl %d tl_wt %d hd %d\n",
-                         wr_rx->w_idx, wr_rx, wr_rx->flags,
-                         wc->opcode == IBV_WC_RDMA_READ ? "RR":"RW_IMM WC",
-                         m_qp->wr_tl_r, m_qp->wr_tl_r_wt, m_qp->wr_hd_r);
-
-               mpxy_lock(&m_qp->rxlock);
-               m_qp->pi_rw_cnt--; /* pending WC post_send */
-               mpxy_unlock(&m_qp->rxlock);
        }
 }
 

diff --git a/dapl/svc/mpxy_out.c b/dapl/svc/mpxy_out.c
index 5c82703027a57d161873ee51ae4aaca813c1de72..1cff1d2fce94e12aba445fe534fca01e9229d1f9 100644 (file)
--- a/dapl/svc/mpxy_out.c
+++ b/dapl/svc/mpxy_out.c
@@ -1100,13 +1100,14 @@ retry:
                       wc[i].vendor_err, wc[i].byte_len, wr_type);
 
                /* Proxy_in -> */
-               if ((wr_type == WRID_RX_RR) || (wr_type == WRID_RX_RW_IMM)) {
+               if (wr_type == WRID_RX_RW_IMM) {  /* RW_imm WC */
+                       continue;
+               }
+               if (wr_type == WRID_RX_RR) {
                        m_wr_rx = (struct mcm_wr_rx *)WRID_ADDR(wc[i].wr_id);
                        assert(m_wr_rx);
                        m_qp = (struct mcm_qp *)m_wr_rx->context;
                        assert(m_qp);
-                       mlog(4," wr_rx_id[%d of %d] wr_rx %p m_qp %p\n",
-                               i+1, ret, m_wr_rx, m_qp);
                        m_pi_req_event(m_qp, m_wr_rx, &wc[i], wr_type);
                        continue;