mpxyd: fix ordering issues with the CCL Proxy receive side forwarding mechanism

scif_writeto doesn't guarantee ordering on DMA posting like IB rdma writes.
Since CCL Proxy is emulating IB semantics we must perserve order of
the rdma write request from MIC consumers via any proxy scif operations.

Changes made to proxy-in to defer forwarding RR completed segments
unless they are middle segments of a larger write operation. On FS or LS
the previous scif_writeto DMA operations must be completed and signaled
before posting a first or last segment. Last segment scif_writeto
operation is ordered to insure last byte is the last byte of
complete rdma write proxied operation.

During scif_wt errors send WC error status for each pending segment
with rdma write operation for accurate proxy-out error processing.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/svc/mpxy_in.c b/dapl/svc/mpxy_in.c
index 54cc62af8a66419ff2083524b5f1a892426e082e..bb48c69cb87736c354d56de604219f4cf46fabde 100644 (file)
--- a/dapl/svc/mpxy_in.c
+++ b/dapl/svc/mpxy_in.c
@@ -476,11 +476,10 @@ static int m_pi_send_wc(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_rx, int status
        int wc_idx, ret;
 
        mlog(0x10,"[%d:%d:%d] WC_rem: wr_rx[%d] %p wc_hd %d flgs %x WR_r tl %d-%d"
-                 " wt %d hd %d wr_id %Lx org_id %Lx\n",
+                 " wt %d hd %d oid %Lx st %d\n",
                m_qp->smd->md->mc->scif_id, m_qp->smd->entry.tid, m_qp->r_entry.tid,
                wr_rx->w_idx, wr_rx, m_qp->wc_hd_rem, wr_rx->flags, m_qp->wr_tl_r,
-               wr_rx->w_idx, m_qp->wr_tl_r_wt, m_qp->wr_hd_r, wr_rx->wr.wr_id,
-               wr_rx->org_id);
+               wr_rx->w_idx, m_qp->wr_tl_r_wt, m_qp->wr_hd_r, wr_rx->org_id, status);
 
        /* local WR and remote WR are serialized, should never reach tail of remote WR */
        if (((m_qp->wc_hd_rem + 1) & m_qp->wrc.wc_end) == m_qp->wc_tl_rem) {
@@ -576,6 +575,13 @@ static void m_pi_post_writeto(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_sig)
        while (m_qp->pi_rr_cnt) { /* RR's pending */
                wr_rx = (struct mcm_wr_rx *)(m_qp->wrc.wr_addr + (m_qp->wrc.wr_sz * wr_idx));
 
+               /* SCIF sync required on IB RW, multiple SCIF writes are not ordered */
+               if (m_qp->post_cnt_wt &&
+                   (wr_rx->flags & (M_SEND_FS|M_SEND_LS)) &&
+                   (!(wr_rx->flags & (M_READ_WRITE_TO_DONE|M_READ_WRITE_TO)))) {
+                       break;
+               }
+
                if (!(wr_rx->flags & M_READ_DONE)) {
                        /* reached head pointer */
                        if (wr_idx == m_qp->wr_hd_r)
@@ -675,8 +681,18 @@ static void m_pi_post_writeto(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_sig)
                        ret = scif_writeto(smd->scif_tx_ep, l_off, w_len, r_off, wt_flag);
 
                        if (ret) {
-                               mlog(0, " ERR: scif_sendto, ret %d err: %d %s\n",
+                               mlog(0," [%d:%d:%d] ERR: scif_sendto, ret %d err: %d %s\n",
+                                       smd->md->mc->scif_id, smd->entry.tid, m_qp->r_entry.tid,
                                        ret, errno, strerror(errno));
+                               mlog(0," PI: wc %d rr %d stall %d wt %d\n",
+                                    m_qp->pi_rw_cnt, m_qp->pi_rr_cnt,
+                                    m_qp->stall_cnt_rr, m_qp->post_cnt_wt);
+                               mlog(0," PO: wr %d wr_rem %d pst_sig %d cmp_sig %d\n",
+                                    m_qp->wr_pp, m_qp->wr_pp_rem,
+                                    m_qp->post_sig_cnt, m_qp->comp_cnt);
+                               mlog(0, " WR_rx[%d] %p l_o %Lx r_o %Lx rb 0x%x-0x%x ln %d id %Lx tl %d hd %d\n",
+                                       wr_rx->w_idx, wr_rx, l_off, r_off, l_start, l_end, w_len, wr_rx->org_id,
+                                       m_qp->wr_tl_r, m_qp->wr_hd_r);
                                goto bail;
                        }
                        MCNTR(smd->md, MCM_SCIF_WRITE_TO);
@@ -719,14 +735,22 @@ static void m_pi_post_writeto(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_sig)
        }
        return;
 bail:
-       /* report error via WC back to proxy-out */
-       mlog(0, " ERR: writeto: wr_rx[%d] %p -> IB raddr %Lx rkey %x"
-               " SCIF r_off %Lx, len %d wr_flags %x wt_pend %d\n",
-               wr_rx->w_idx, wr_rx, wr_rx->wr.wr.rdma.remote_addr,
-               wr_rx->wr.wr.rdma.rkey, r_off, sg_len, wr_rx->flags,
-               m_qp->post_cnt_wt);
-
-       m_pi_send_wc(m_qp, wr_rx, IBV_WC_REM_ACCESS_ERR);
+       /* report error via WC back to proxy-out, all pending WRs */
+       wr_idx = m_qp->wr_tl_r_wt;
+       do {
+               wr_rx = (struct mcm_wr_rx *)(m_qp->wrc.wr_addr + (m_qp->wrc.wr_sz * wr_idx));
+
+               mlog(0, " ERR: wr_rx[%d] %p -> IB raddr %Lx %x"
+                       " SCIF r_o %Lx, ln %d fl %x wt_pnd %d\n",
+                       wr_rx->w_idx, wr_rx, wr_rx->wr.wr.rdma.remote_addr,
+                       wr_rx->wr.wr.rdma.rkey, r_off, sg_len, wr_rx->flags,
+                       m_qp->post_cnt_wt);
+
+               m_pi_send_wc(m_qp, wr_rx, IBV_WC_REM_ACCESS_ERR);
+               wr_idx = (wr_idx + 1) & m_qp->wrc.wr_end; /* next WR */
+
+       } while (wr_idx != m_qp->wr_hd_r);
+
        return;
 }
 
@@ -1164,6 +1188,17 @@ retry:
        goto retry;
 }
 
+/* Process scif_writeto DMAs waiting for previous WTs completions, order */
+void m_pi_pending_wt(struct mcm_qp *m_qp)
+{
+       struct mcm_wr_rx *wr_rx;
+
+       mpxy_lock(&m_qp->rxlock);
+       wr_rx = (struct mcm_wr_rx *)(m_qp->wrc.wr_addr + (m_qp->wrc.wr_sz * m_qp->wr_hd_r));
+       m_pi_post_writeto(m_qp, wr_rx);
+       mpxy_unlock(&m_qp->rxlock);
+}
+
 /*
  * Pending Proxy-in services for RDMA Writes from remote peer
  *

diff --git a/dapl/svc/mpxyd.c b/dapl/svc/mpxyd.c
index 922eeae693d139e209750256bbe78ead0070d99e..4269ff9a05d69369849b1e3727b4b11e470b6be0 100644 (file)
--- a/dapl/svc/mpxyd.c
+++ b/dapl/svc/mpxyd.c
@@ -1187,6 +1187,7 @@ void mpxy_rx_thread(void *mic_client)
                                m_qp = get_head_entry(&smd->qprlist);
                                while (m_qp) {
                                        m_pi_pending_wr(m_qp, &data); /* RR's and scif_sendto */
+                                       m_pi_pending_wt(m_qp); /* WT's pending */
                                        m_qp = get_next_entry(&m_qp->r_entry, &smd->qprlist);
                                }
                                mpxy_unlock(&smd->qprlock);

diff --git a/dapl/svc/mpxyd.h b/dapl/svc/mpxyd.h
index c733157d974ed89527b75b31407d55a837b10caa..8addaa2ac1febd2715f3543aa499fce59b8f8bc8 100644 (file)
--- a/dapl/svc/mpxyd.h
+++ b/dapl/svc/mpxyd.h
@@ -613,6 +613,7 @@ int m_pi_create_sr_q(struct mcm_qp *m_qp, int entries);
 int m_pi_create_bpool(struct mcm_qp *m_qp, int max_recv_wr);
 void m_qp_destroy_pi(struct mcm_qp *m_qp);
 int m_qp_create_pi(mcm_scif_dev_t *smd, struct mcm_qp *m_qp);
+void m_pi_pending_wt(struct mcm_qp *m_qp);
 void m_pi_pending_wr(struct mcm_qp *m_qp, int *data);
 void m_pi_pending_wc(struct mcm_qp *m_qp, int *events);
 void m_pi_req_event(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_rx, struct ibv_wc *wc, int type);