From: Arlin Davis <arlin.r.davis@intel.com>
Date: Mon, 24 Feb 2014 17:03:15 +0000 (-0800)
Subject: mpxyd: sync PI WC trigger to PO MP_SIG
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=11494cfd464198f57f993289804fbc4487030841;p=~ardavis%2Fdapl.git

mpxyd: sync PI WC trigger to PO MP_SIG

The PI and PO segment comp signal was out of sync so the WC update
back from the PI incorrectly updated the m_po_buf_tl
on the PO side. Set signaling on both sides based
on the rdma_write initiator setting via the WR M_SEND_MP_SIG bit.

Modify some log levels and add check for m_idx during tail update.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>
---

diff --git a/dapl/svc/mcm.c b/dapl/svc/mcm.c
index 3bafedd..1f300f3 100644
--- a/dapl/svc/mcm.c
+++ b/dapl/svc/mcm.c
@@ -1319,15 +1319,15 @@ retry:
 
 		/* validate CM message, version */
 		if (ntohs(msg->ver) != DAT_MCM_VER) {
-			mlog(1, " UNKNOWN cm_msg %p, op %d ver %d st %x ln %d\n",
-				msg, ntohs(msg->op), ntohs(msg->ver),
+			mlog(1, " UNKNOWN cm_msg %p, op %s ver %d st %x ln %d\n",
+				msg, mcm_op_str(ntohs(msg->op)), ntohs(msg->ver),
 				wc[i].status, wc[i].byte_len);
 			mcm_post_rmsg(md, msg);
 			continue;
 		}
 		if (!(cm = mcm_get_cm(md, msg))) {
-			mlog(1, " NO_MATCH cm_msg %p, op %d ver %d st %x ln %d\n",
-				msg, ntohs(msg->op), ntohs(msg->ver),
+			mlog(2, " NO_MATCH cm_msg %p, op %s ver %d st %x ln %d\n",
+				msg, mcm_op_str(ntohs(msg->op)), ntohs(msg->ver),
 				wc[i].status, wc[i].byte_len);
 			mcm_post_rmsg(md, msg);
 			continue;
diff --git a/dapl/svc/mpxy_in.c b/dapl/svc/mpxy_in.c
index 6ba736e..6446402 100644
--- a/dapl/svc/mpxy_in.c
+++ b/dapl/svc/mpxy_in.c
@@ -254,8 +254,11 @@ static void m_pi_buf_hd(mcm_scif_dev_t *smd, int m_idx)
 		smd->m_buf_hd_r, m_idx);
 
 	if (smd->m_buf_hd_r == smd->m_buf_tl_r) {
-		mlog(0, " WARNING: RX Proxy Buffer WC out of slots\n");
-		return; /* todo, add support to wait */
+		mlog(0," ERR: PI Buf WC full (%d) m_buf_wc_r %p:"
+			"tl %d hd %d buf_wc_hd_r[%d].m_idx=0x%x\n",
+			smd->m_buf_end_r, smd->m_buf_wc_r, smd->m_buf_tl_r,
+			smd->m_buf_hd_r, smd->m_buf_hd_r, m_idx);
+		return;
 	}
 	smd->m_buf_wc_r[smd->m_buf_hd_r].m_idx = m_idx;
 	smd->m_buf_wc_r[smd->m_buf_hd_r].done = 0;
@@ -1245,8 +1248,8 @@ void m_pi_pending_wr(struct mcm_qp *m_qp, int *data)
 				m_pi_free_sr(m_qp, m_sr);
 
 			}
-			/* need regular WC's to release PI buffers */
-			if ((wr_rx->flags & M_SEND_LS) || (!(wr_rx->w_idx % 10))) {
+			/* Last Segment or peer PO wants signaled */
+			if ((wr_rx->flags & M_SEND_LS) || (wr_rx->flags & M_SEND_MP_SIG)) {
 				mlog(4, "WR_rx[%d] wr %p LastSeg: send WC! tl %d hd %d\n",
 					wr_rx->w_idx, wr_rx, m_qp->wr_tl_r, m_qp->wr_hd_r);
 
diff --git a/dapl/svc/mpxy_out.c b/dapl/svc/mpxy_out.c
index 9b22327..56ba400 100644
--- a/dapl/svc/mpxy_out.c
+++ b/dapl/svc/mpxy_out.c
@@ -89,15 +89,16 @@ err:
 /* called with smd->sblock */
 static void m_po_buf_hd(mcm_scif_dev_t *smd, int m_idx)
 {
-	mlog(8," m_hd 0x%Lx - m_buf_wc %p: hd %d tl %d buf_wc_r[%d].m_idx=0x%x\n",
+	mlog(4," m_hd 0x%Lx - m_buf_wc %p: hd %d tl %d buf_wc[%d].m_idx=0x%x\n",
 		smd->m_hd, smd->m_buf_wc, smd->m_buf_hd, smd->m_buf_tl,
 		smd->m_buf_hd, m_idx);
 
 	if (smd->m_buf_hd == smd->m_buf_tl) {
-		mlog(0, " WARNING: TX Proxy Buffer WC out of slots\n");
-		mlog(0," m_buf_wc %p: hd %d tl %d buf_wc_r[%d].m_idx=0x%x\n",
-			smd->m_buf_wc, smd->m_buf_hd, smd->m_buf_tl,
-			smd->m_buf_hd, m_idx);
+		mlog(0, " ERR: PO Buf WC full (%d) m_buf_wc %p:"
+			"tl %d hd %d buf_wc_hd[%d].m_idx=0x%x\n",
+			smd->m_buf_end, smd->m_buf_wc, smd->m_buf_tl,
+			smd->m_buf_hd, smd->m_buf_hd, m_idx);
+		return;
 	}
 	smd->m_buf_wc[smd->m_buf_hd].m_idx = m_idx;
 	smd->m_buf_wc[smd->m_buf_hd].done = 0;
@@ -115,7 +116,13 @@ static void m_po_buf_tl(mcm_scif_dev_t *smd, int m_idx)
 	if (idx == smd->m_buf_end)
 		idx = 0;
 
-	mlog(8," m_tl 0x%Lx -> m_idx 0x%x buf_wc[%d].m_idx=0x%x\n",
+	if (m_idx == 0) {
+		mlog(1," WARN: m_idx = 0: m_tl 0x%Lx -> m_idx 0x%x buf_wc[%d].m_idx=0x%x\n",
+			smd->m_tl, m_idx, idx, smd->m_buf_wc[idx].m_idx);
+		return;
+	}
+
+	mlog(4," m_tl 0x%Lx -> m_idx 0x%x buf_wc[%d].m_idx=0x%x\n",
 		smd->m_tl, m_idx, idx, smd->m_buf_wc[idx].m_idx);
 
 	/* 1st one, completed in order, move proxy buffer tl and buf wc tl */
@@ -551,7 +558,7 @@ retry_mr:
 			if (last_rf) {
 				uint32_t now = mcm_ts_us();
 				if ((now - last_rf) > 500) {
-					mlog(0x10, " WARN: delayed post (%d us):"
+					mlog(0x4, " WARN: delayed post (%d us):"
 						   " WR[%d] hd %d tl %d io %d"
 						   " pst %d,%d cmp %d wr_pp %d\n",
 						now - last_rf, m_wr->w_idx,
@@ -754,11 +761,13 @@ void m_po_wc_event(struct mcm_qp *m_qp, struct mcm_wc_rx *wc_rx, int wc_idx)
 		m_wr->w_idx, m_qp->wr_hd, m_qp->wc_tl, wc_idx,
 		m_qp->post_cnt, m_qp->post_sig_cnt, m_qp->comp_cnt);
 
-	/* PI peer has copy of data, free buffer slot */
-	mpxy_lock(&m_qp->smd->tblock);
-	m_po_buf_tl(m_qp->smd, m_wr->m_idx); /* move proxy buffer tail */
-	m_wr->m_idx = 0;
-	mpxy_unlock(&m_qp->smd->tblock);
+	/* PI peer has copy of data, free buffer slot if saved on MP signal mark */
+	if (m_wr->flags & M_SEND_MP_SIG) {
+		mpxy_lock(&m_qp->smd->tblock);
+		m_po_buf_tl(m_qp->smd, m_wr->m_idx); /* move proxy buffer tail */
+		m_wr->m_idx = 0;
+		mpxy_unlock(&m_qp->smd->tblock);
+	}
 
 	if (m_qp->wr_addr_rem)  /* remote MXS: sync PO WR tail with remote PI WR tail */
 		m_qp->wr_tl = wc_rx->wr_tl;