From 110bb16d9ad9436f035e00d928a7902e84055b59 Mon Sep 17 00:00:00 2001 From: Jianxin Xiong Date: Fri, 17 Aug 2012 11:35:25 -0700 Subject: [PATCH] Minor code cleanup. --- dapl/svc/mpxyd.c | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/dapl/svc/mpxyd.c b/dapl/svc/mpxyd.c index deeb47d..10b9e59 100644 --- a/dapl/svc/mpxyd.c +++ b/dapl/svc/mpxyd.c @@ -2622,7 +2622,7 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) int len, ret, i, wr_idx, pending; off_t l_off, r_off; uint64_t total_offset; - int l_start, l_end, l_len, cacheln_off, seg_len, num_sge, total_len, wr_enqueued; + int l_start, l_end, l_len, cacheln_off, seg_len; struct mcm_qp *m_qp; struct ibv_send_wr *m_wr; struct ibv_sge *m_sge; @@ -2710,6 +2710,8 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) } #else + total_offset = 0; + m_wr = (struct ibv_send_wr *)(m_qp->wr_buf + (DAT_MCM_WR * m_qp->wr_hd)); m_sge = (struct ibv_sge *)((char*)m_wr + 72); @@ -2720,11 +2722,8 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) m_wr->sg_list = m_sge; m_wr->next = 0; m_wr->xrc_remote_srq_num = 0; - - num_sge = 0; - total_len = 0; - total_offset = 0; - wr_enqueued = 0; + m_wr->wr.rdma.remote_addr += total_offset; + m_wr->num_sge = 0; for (i=0;iwr.num_sge;i++) { @@ -2735,7 +2734,8 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) r_off = ALIGN_DOWN_64(r_off); while (l_len) { - num_sge ++; + m_wr->num_sge ++; + seg_len = (l_len > m_qp->m_seg)? m_qp->m_seg : l_len; l_start = ALIGN_64(m_qp->m_hd); @@ -2763,7 +2763,7 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) m_qp->m_hd = l_end; m_sge->addr = (uint64_t)(m_qp->m_buf + l_start + cacheln_off); - m_sge->length = seg_len-cacheln_off; + m_sge->length = seg_len - cacheln_off; if (m_sge->length > len) m_sge->length = len; m_sge->lkey = m_qp->m_mr->lkey; @@ -2772,9 +2772,10 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) l_len -= seg_len; r_off += seg_len; - total_len += m_sge->length; len -= m_sge->length; + total_offset += m_sge->length; cacheln_off = 0; /* only apply to the first segment of a sge */ + m_sge++; /* if enough for this WR, then set up DMA signal, and move to next WR */ @@ -2791,11 +2792,6 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) return -1; } - /* update the WR with the corret info */ - m_wr->num_sge = num_sge; - m_wr->wr.rdma.remote_addr += total_offset; - total_offset = total_len; - /* remove special flags unless it's the last segment */ if (l_len || i != pmsg->wr.num_sge -1) { if (m_wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM) @@ -2803,15 +2799,15 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) m_wr->send_flags &= IBV_SEND_INLINE; } - wr_enqueued++; + *data = *data + 1; /* took WR slot, move head */ if (++m_qp->wr_hd == m_qp->wr_end) m_qp->wr_hd = 0; + /* prepare the next WR */ m_wr = (struct ibv_send_wr *)(m_qp->wr_buf + (DAT_MCM_WR * m_qp->wr_hd)); m_sge = (struct ibv_sge *)((char*)m_wr + 72); - num_sge = 0; mlog(1, " m_wr %p m_sge %p \n", m_wr, m_sge); @@ -2820,16 +2816,17 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) m_wr->sg_list = m_sge; m_wr->next = 0; m_wr->xrc_remote_srq_num = 0; + m_wr->wr.rdma.remote_addr += total_offset; + m_wr->num_sge = 0; } } } - wr_idx = m_qp->wr_tl; - while (wr_idx != m_qp->wr_hd) { - m_wr = (struct ibv_send_wr *)(m_qp->wr_buf + (DAT_MCM_WR * wr_idx)); + while (m_qp->wr_tl != m_qp->wr_hd) { + m_wr = (struct ibv_send_wr *)(m_qp->wr_buf + (DAT_MCM_WR * m_qp->wr_tl)); /* if read data already arrived (via scif_signal) post send */ - mlog(1, " m_wr wr_id %p == m_sge->lkey %p ? \n", m_wr->wr_id, m_wr->wr.atomic.swap); + mlog(1, " m_wr wr_id %p == pmsg->wr.wr_id %p ? \n", m_wr->wr_id, m_wr->wr.atomic.swap); if (m_wr->wr_id == m_wr->wr.atomic.swap) { char *sbuf = (char*)m_wr->sg_list->addr; mlog(1, " m_wr %p data ready for IB write, 1st byte 0x%x last byte 0x%x \n", @@ -2852,16 +2849,12 @@ static int mix_post_write(mcm_scif_dev_t *smd, dat_mix_send_t *pmsg, int *data) mix_dto_event(m_qp->ib_qp->send_cq->cq_context, &wc, 1); } m_wr->wr_id = 0; - if (++m_qp->wr_tl == m_qp->wr_end) /* move tail */ + if (++m_qp->wr_tl == m_qp->wr_end) /* WR posted, move tail */ m_qp->wr_tl = 0; - if (++wr_idx == m_qp->wr_end) /* posted WR, move tail */ - wr_idx = 0; - - wr_enqueued--; + *data = *data - 1; } else { - *data = *data + wr_enqueued; - mlog(1, " wr_id %p, data not ready, %d WR's pending, wr_hd=%d\n", pmsg->wr.wr_id, *data, wr_idx); + mlog(1, " wr_id %p, data not ready, %d WR's pending, wr_hd=%d\n", pmsg->wr.wr_id, *data, m_qp->wr_hd); break; } } -- 2.41.0