mpxyd: append _free to dqconn/dqlisten for readability, improve logging

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/svc/mcm.c b/dapl/svc/mcm.c
index 63afa9ae09c78408a6a8cde4e5b99e4e8a92ac60..e6207e798118e4d678120ac6b374810c37e77586 100644 (file)
--- a/dapl/svc/mcm.c
+++ b/dapl/svc/mcm.c
@@ -323,11 +323,11 @@ int mcm_modify_qp(struct ibv_qp   *qp_handle,
                qp_attr.ah_attr.port_num = m_qp->smd->md->port;
 
                mlog(8, " QPS_RTR: l_qpn %x type %d r_qpn 0x%x gid %p (%d) lid 0x%x"
-                       " port %d ep %p qp_state %d rd_atomic %d rnr_timer %d\n",
+                       " port %d qp %p qp_state %d rd_atomic %d rnr_timer %d\n",
                        qp_handle->qp_num, qp_handle->qp_type,
                        ntohl(qpn), gid, m_qp->smd->md->dev_attr.global,
-                       ntohs(lid), m_qp->smd->md->port, m_qp,
-                       m_qp->qp_attr2.cur_state, qp_attr.max_dest_rd_atomic,
+                       ntohs(lid), m_qp->smd->md->port, qp_handle,
+                       qp_handle->state, qp_attr.max_dest_rd_atomic,
                        qp_attr.min_rnr_timer);
                break;
 
@@ -347,8 +347,8 @@ int mcm_modify_qp(struct ibv_qp     *qp_handle,
                        " retry %d rnr_retry %d qpn %x qp_state %d\n",
                        qp_attr.sq_psn, qp_attr.max_rd_atomic,
                        qp_attr.timeout, qp_attr.retry_cnt,
-                       qp_attr.rnr_retry, m_qp->ib_qp2->qp_num,
-                       m_qp->qp_attr2.cur_state);
+                       qp_attr.rnr_retry, qp_handle->qp_num,
+                       qp_handle->state);
                break;
 
        case IBV_QPS_INIT:
@@ -591,19 +591,21 @@ void mcm_qconn(mcm_scif_dev_t *smd, mcm_cm_t *cm)
        mpxy_unlock(&smd->clock);
 }
 /* dequeue connection object from CM list */
-void mcm_dqconn(mcm_scif_dev_t *smd, mcm_cm_t *cm)
+void mcm_dqconn_free(mcm_scif_dev_t *smd, mcm_cm_t *cm)
 {
        /* Remove from work queue, cr thread processing */
-       mpxy_lock(&smd->clock);
-       remove_entry(&cm->entry);
-       mpxy_unlock(&smd->clock);
-
-       mpxy_lock(&cm->lock);
-       cm->ref_cnt--; /* clist ref */
-       mpxy_unlock(&cm->lock);
+       if (cm->entry.tid) {
+               mpxy_lock(&smd->clock);
+               remove_entry(&cm->entry);
+               mpxy_unlock(&smd->clock);
 
+               mpxy_lock(&cm->lock);
+               cm->ref_cnt--; /* clist ref */
+               mpxy_unlock(&cm->lock);
+       }
        m_cm_free(cm);
 }
+
 /* queue listen object on listen list */
 void mcm_qlisten(mcm_scif_dev_t *smd, mcm_cm_t *cm)
 {
@@ -617,21 +619,22 @@ void mcm_qlisten(mcm_scif_dev_t *smd, mcm_cm_t *cm)
        mpxy_unlock(&smd->llock);
 }
 /* dequeue listen object from listen list */
-void mcm_dqlisten(mcm_scif_dev_t *smd, mcm_cm_t *cm)
+void mcm_dqlisten_free(mcm_scif_dev_t *smd, mcm_cm_t *cm)
 {
-       mpxy_lock(&smd->llock);
-       remove_entry(&cm->entry);
-       mpxy_unlock(&smd->llock);
-
-       mpxy_lock(&smd->md->plock);
-       mcm_free_port(smd->md->ports, cm->sid);
-       cm->sid = 0;
-       mpxy_unlock(&smd->md->plock);
+       if (cm->entry.tid) {
+               mpxy_lock(&smd->llock);
+               remove_entry(&cm->entry);
+               mpxy_unlock(&smd->llock);
 
-       mpxy_lock(&cm->lock);
-       cm->ref_cnt--; /* llist ref */
-       mpxy_unlock(&cm->lock);
+               mpxy_lock(&smd->md->plock);
+               mcm_free_port(smd->md->ports, cm->sid);
+               cm->sid = 0;
+               mpxy_unlock(&smd->md->plock);
 
+               mpxy_lock(&cm->lock);
+               cm->ref_cnt--; /* llist ref */
+               mpxy_unlock(&cm->lock);
+       }
        m_cm_free(cm);
 }
 
@@ -1004,8 +1007,8 @@ int mcm_cm_rep_out(mcm_cm_t *cm)
 
 static void mcm_process_recv(mcm_ib_dev_t *md, dat_mcm_msg_t *msg, mcm_cm_t *cm, int len)
 {
-       mlog(2, " cm %p cm_id %d state %s \n",
-               cm, cm->entry.tid, mcm_state_str(cm->state));
+       mlog(2, " cm %p cm_id %d state %s cm->m_qp %p\n",
+               cm, cm->entry.tid, mcm_state_str(cm->state), cm->m_qp);
        mpxy_lock(&cm->lock);
        switch (cm->state) {
        case MCM_LISTEN: /* passive */

diff --git a/dapl/svc/mix.c b/dapl/svc/mix.c
index a66052ff6c101012d0bd9f74b8ccc6863f2b9969..db59f9b066fe913299ffc0644a0e9c93d605290d 100644 (file)
--- a/dapl/svc/mix.c
+++ b/dapl/svc/mix.c
@@ -214,7 +214,7 @@ static int mix_listen_free(mcm_scif_dev_t *smd, dat_mix_hdr_t *pmsg)
        mpxy_unlock(&smd->llock);
 
        if (cm) {
-               mcm_dqlisten(smd, cm);
+               mcm_dqlisten_free(smd, cm);
                pmsg->status = MIX_SUCCESS;
        } else {
                mlog(0, " MIX_LISTEN_FREE: ERR: sid 0x%x not found\n", pmsg->req_id);
@@ -468,7 +468,8 @@ static int mix_cq_create(mcm_scif_dev_t *smd, dat_mix_cq_t *pmsg)
        }
 
        cq_len = max(pmsg->cq_len, mcm_tx_cq_size);
-       mlog(8, " MIX_CQ_CREATE: cq_len = %d,%d mic_ctx = %Lx\n", pmsg->cq_len, cq_len, pmsg->cq_ctx);
+       mlog(8, " MIX_CQ_CREATE: cq_len = %d,%d mic_ctx = %Lx, dev_id %d\n",
+               pmsg->cq_len, cq_len, pmsg->cq_ctx, smd->entry.tid);
 
        if (m_cq_create(smd, cq_len, &new_mcq, MCM_CQ_TX))
                goto err;
@@ -514,7 +515,7 @@ void m_qp_free(struct mcm_qp *m_qp)
                m_qp->cm->m_qp = NULL;
                m_qp->cm = NULL;
                mpxy_unlock(&cm->lock);
-               mcm_dqconn(m_qp->smd, cm);
+               mcm_dqconn_free(m_qp->smd, cm);
        }
        mcm_flush_qp(m_qp); /* move QP to error, flush & destroy */
 
@@ -790,8 +791,9 @@ static int mix_qp_create(mcm_scif_dev_t *smd, dat_mix_qp_t *pmsg)
                return -1;
        }
 
-       mlog(8, " Client QP_r - qpn 0x%x, ctx %p, rq %d,%d rcq_id %d\n",
-               pmsg->qp_r.qp_num, pmsg->qp_r.ctx, pmsg->qp_r.max_recv_wr,
+       mlog(8, " Client QP_r - dev_id %d qpn 0x%x, ctx %p, rq %d,%d rcq_id %d\n",
+               smd->entry.tid, pmsg->qp_r.qp_num, pmsg->qp_r.ctx,
+               pmsg->qp_r.max_recv_wr,
                pmsg->qp_r.max_recv_sge, pmsg->qp_r.rcq_id);
 
        mlog(8, " Client QP_t - wr %d sge %d inline %d sq %d,%d scq_id %d\n",
@@ -1010,8 +1012,8 @@ void mix_cm_event(mcm_cm_t *m_cm, uint32_t event)
                mpxy_unlock(&m_cm->lock);
        }
 
-       mlog(2, " MIX_CM_EVENT: cm %p cm_id %d, ctx %p, event 0x%x\n",
-            m_cm, m_cm->entry.tid, msg.cm_ctx, event);
+       mlog(2, " MIX_CM_EVENT: cm %p cm_id %d, ctx %p, event 0x%x dev_id %d\n",
+            m_cm, m_cm->entry.tid, msg.cm_ctx, event, m_cm->smd->entry.tid);
 
        len = sizeof(dat_mix_cm_event_t);
        mpxy_lock(&m_cm->smd->evlock);
@@ -1102,8 +1104,9 @@ err:
 
        pmsg->hdr.status = MIX_EINVAL;
 resp:
-       mlog(8, " MPXYD id 0x%x, ctx %p - MIC id 0x%x, ctx %p\n",
-            pmsg->cm_id, pmsg->cm_ctx, m_cm->cm_id, m_cm->cm_ctx);
+       mlog(8, " MPXYD id 0x%x, ctx %p - MIC id 0x%x, ctx %p dev_id %d\n",
+            pmsg->cm_id, pmsg->cm_ctx, m_cm->cm_id, m_cm->cm_ctx,
+            smd->entry.tid);
 
        /* send back response */
        pmsg->hdr.flags = MIX_OP_RSP;
@@ -1153,8 +1156,8 @@ static int mix_cm_rtu_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
                return ret;
        }
-       mlog(8, " cm_id %d, cm_ctx %p, qp_id %d \n",
-               pmsg->cm_id, (void*)pmsg->cm_ctx, pmsg->qp_id);
+       mlog(8, " cm_id %d, cm_ctx %p, qp_id %d dev_id\n",
+               pmsg->cm_id, (void*)pmsg->cm_ctx, pmsg->qp_id, smd->entry.tid);
 
        /* Find the CM for linking */
        m_cm = mix_get_cm(smd, pmsg->cm_id);
@@ -1182,8 +1185,8 @@ int mix_cm_rej_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
        dat_mix_cm_t msg;
        int len;
 
-       mlog(2, " cm_id %d, ctx %p, m_cm %p pkt %p pln %d psz %d r_guid %Lx\n",
-            m_cm->cm_id, m_cm->cm_ctx, m_cm, pkt,
+       mlog(2, " dev_id %d cm_id %d, ctx %p, m_cm %p pkt %p pln %d psz %d r_guid %Lx\n",
+            m_cm->smd->entry.tid, m_cm->cm_id, m_cm->cm_ctx, m_cm, pkt,
             pkt_len, ntohs(pkt->p_size), ntohll(pkt->sys_guid));
 
        /* Forward appropriate reject message to MIC client */
@@ -1242,8 +1245,8 @@ int mix_cm_rep_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
 
        MCNTR(m_cm->md, MCM_CM_REP_IN);
 
-       mlog(2, " cm_id %d, ctx %p, m_cm %p pkt %p pln %d psz %d r_guid %Lx\n",
-            m_cm->cm_id, m_cm->cm_ctx, m_cm, pkt,
+       mlog(2, " dev_id %d cm_id %d, ctx %p, m_cm %p pkt %p pln %d psz %d r_guid %Lx\n",
+            m_cm->smd->entry.tid, m_cm->cm_id, m_cm->cm_ctx, m_cm, pkt,
             pkt_len, ntohs(pkt->p_size), ntohll(pkt->sys_guid));
 
        /* Forward, as is, conn_reply message to MIC client, with remote QP info */
@@ -1265,6 +1268,11 @@ int mix_cm_rep_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
        memcpy(&m_cm->msg.daddr2, &pkt->saddr2, sizeof(dat_mcm_addr_t));
        mcm_ntoh_wrc(&m_cm->m_qp->wrc_rem, (mcm_wrc_info_t *)m_cm->msg.p_proxy); /* peer RI WRC info */
 
+       mlog(2, " WRC_rem: m_qp %p - addr 0x%Lx rkey 0x%x len %d, sz %d end %d\n",
+            m_cm->m_qp, m_cm->m_qp->wrc_rem.wc_addr, m_cm->m_qp->wrc_rem.wc_rkey,
+            m_cm->m_qp->wrc_rem.wc_len, m_cm->m_qp->wrc_rem.wc_sz,
+            m_cm->m_qp->wrc_rem.wc_end);
+
        /* MXS <- MSS or HOST, fabric: TX: QP2->QP1 direct, RX: QP1<-QP2 proxy */
        if ((MXS_EP(&m_cm->md->addr) && !MXS_EP(&m_cm->msg.daddr1)) &&
            system_guid != m_cm->msg.sys_guid) {
@@ -1444,7 +1452,7 @@ int mix_cm_rtu_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
        dat_mix_cm_t msg;
        int len;
 
-       mlog(1, " CONNECTED[%d]: %p 0x%x %x 0x%x %Lx %s <- 0x%x %x 0x%x %Lx %s\n",
+       mlog(1, " CONNECTED[%d]: %p 0x%x %x 0x%x %Lx %s <- 0x%x %x 0x%x %Lx %s dev_id %d\n",
                m_cm->md->cntrs ? (uint32_t)((uint64_t *)m_cm->md->cntrs)[MCM_CM_RTU_IN]:0,
                m_cm, htons(pkt->daddr1.lid),
                MXS_EP(&m_cm->msg.daddr1) && MXS_EP(&m_cm->msg.saddr1) ?
@@ -1452,11 +1460,16 @@ int mix_cm_rtu_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
                htons(pkt->dport), system_guid, mcm_map_str(pkt->daddr1.ep_map),
                htons(pkt->saddr2.lid), htonl(pkt->saddr2.qpn),
                htons(pkt->sport), ntohll(pkt->sys_guid),
-               mcm_map_str(pkt->saddr2.ep_map));
+               mcm_map_str(pkt->saddr2.ep_map), m_cm->smd->entry.tid);
 
        /* MXS_EP <- HST_EP, host sends WC on RTU, save WRC info */
-       if (MXS_EP(&pkt->daddr1) && HST_EP(&pkt->saddr2))
+       if (MXS_EP(&pkt->daddr1) && HST_EP(&pkt->saddr2)) {
                mcm_ntoh_wrc(&m_cm->m_qp->wrc_rem, (mcm_wrc_info_t *)pkt->p_proxy);
+               mlog(2, " WRC_rem: m_qp %p - addr 0x%Lx rkey 0x%x len %d, sz %d end %d\n",
+                    m_cm->m_qp, m_cm->m_qp->wrc_rem.wc_addr, m_cm->m_qp->wrc_rem.wc_rkey,
+                    m_cm->m_qp->wrc_rem.wc_len, m_cm->m_qp->wrc_rem.wc_sz,
+                    m_cm->m_qp->wrc_rem.wc_end);
+       }
 
        /* Forward, as is, conn_reply message to MIC client, with remote QP info */
        msg.hdr.ver = DAT_MIX_VER;
@@ -1494,8 +1507,9 @@ static int mix_cm_rep_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
                return ret;
        }
-       mlog(8, "mic_ctx %p my_id %d, my_ctx %p, qp_id %d, pmsg_sz=%d \n",
-               pmsg->sp_ctx, pmsg->cm_id, (void*)pmsg->cm_ctx, pmsg->qp_id, sizeof(dat_mix_cm_t));
+       mlog(8, "dev_id %d mic_ctx %p my_id %d, my_ctx %p, qp_id %d, pmsg_sz=%d \n",
+               smd->entry.tid, pmsg->sp_ctx, pmsg->cm_id,
+               (void*)pmsg->cm_ctx, pmsg->qp_id, sizeof(dat_mix_cm_t));
 
        /* Find the CM for this reply */
        m_cm = mix_get_cm(smd, pmsg->cm_id);
@@ -1521,6 +1535,10 @@ static int mix_cm_rep_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        m_cm->ref_cnt++; /* Passive: QP ref */
        m_cm->m_qp->cm = m_cm;
        mcm_ntoh_wrc(&m_cm->m_qp->wrc_rem, (mcm_wrc_info_t *)m_cm->msg.p_proxy); /* save peer PI WRC info */
+       mlog(2, " WRC_rem: m_qp %p - addr 0x%Lx rkey 0x%x len %d, sz %d end %d\n",
+            m_cm->m_qp, m_cm->m_qp->wrc_rem.wc_addr, m_cm->m_qp->wrc_rem.wc_rkey,
+            m_cm->m_qp->wrc_rem.wc_len, m_cm->m_qp->wrc_rem.wc_sz,
+            m_cm->m_qp->wrc_rem.wc_end);
 
        mlog(8, " loc_guid %Lx, rem_guid %Lx\n",
                ntohll(system_guid), ntohll(m_cm->msg.sys_guid));
@@ -1700,8 +1718,8 @@ static int mix_cm_rej_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
                mlog(0, " ERR: mix_get_cm, id %d, not found\n", pmsg->cm_id);
                return -1;
        }
-       mlog(2, " MIX_CM_REJ_OUT: cm_id %d, cm_ctx %p, qp_id %d m_cm %p\n",
-               pmsg->cm_id, (void*)pmsg->cm_ctx, pmsg->qp_id, m_cm);
+       mlog(2, " MIX_CM_REJ_OUT: dev_id %d cm_id %d, cm_ctx %p, qp_id %d m_cm %p\n",
+               smd->entry.tid, pmsg->cm_id, (void*)pmsg->cm_ctx, pmsg->qp_id, m_cm);
 
        /* update CM message from MIX client, send back guid */
        memcpy(&m_cm->msg, &pmsg->msg, sizeof(dat_mcm_msg_t));
@@ -1714,7 +1732,7 @@ static int mix_cm_rej_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        /* send rej on wire */
        m_cm->state = MCM_REJECTED;
        mcm_cm_rej_out(smd->md, &m_cm->msg, MCM_REJ_USER, 0);
-       mcm_dqconn(smd, m_cm); /* dequeue and free */
+       mcm_dqconn_free(smd, m_cm); /* dequeue and free */
        return 0;
 }
 
@@ -1970,9 +1988,9 @@ int mix_scif_recv(mcm_scif_dev_t *smd, scif_epd_t scif_ep)
                     phdr->flags);
                return -1;
        }
-       mlog(2, " <--- scif_ep %d %d bytes: ver %d, op %s, flgs %d id %d\n",
-            scif_ep, len, phdr->ver, mix_op_str(phdr->op), phdr->flags,
-            phdr->req_id);
+       mlog(8, " dev_id %d scif_ep %d, %d bytes: ver %d, op %s, flgs %d id %d\n",
+            smd->entry.tid, scif_ep, len, phdr->ver, mix_op_str(phdr->op),
+            phdr->flags, phdr->req_id);
 
        switch (phdr->op) {
        case MIX_MR_CREATE:

diff --git a/dapl/svc/mpxy_in.c b/dapl/svc/mpxy_in.c
index e3451dbac85588ad6c8e3e554f864c5740fea1c9..7bbad0375d0c1f164cee4fa95bb4051ae96cc54a 100644 (file)
--- a/dapl/svc/mpxy_in.c
+++ b/dapl/svc/mpxy_in.c
@@ -439,9 +439,11 @@ static int m_pi_send_wc(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_rx, int status
        struct ibv_qp *ib_qp;
        int wc_idx, ret;
 
-       mlog(0x10," WC_rem: wr_rx[%d] %p, wc_idx %d flgs 0x%x, WR_r tl %d-%d wt %d hd %d\n",
+       mlog(0x10," WC_rem: wr_rx[%d] %p wc_hd %d flgs %x WR_r tl %d-%d"
+                 " wt %d hd %d wr_id %Lx org_id %Lx\n",
                wr_rx->w_idx, wr_rx, m_qp->wc_hd_rem, wr_rx->flags,
-               m_qp->wr_tl_r, wr_rx->w_idx, m_qp->wr_tl_r_wt, m_qp->wr_hd_r);
+               m_qp->wr_tl_r, wr_rx->w_idx, m_qp->wr_tl_r_wt, m_qp->wr_hd_r,
+               wr_rx->wr.wr_id, wr_rx->org_id);
 
        /* local WR and remote WR are serialized, should never reach tail of remote WR */
        if (((m_qp->wc_hd_rem + 1) & m_qp->wrc.wc_end) == m_qp->wc_tl_rem) {
@@ -505,7 +507,9 @@ static int m_pi_send_wc(struct mcm_qp *m_qp, struct mcm_wr_rx *wr_rx, int status
        }
        m_qp->pi_rw_cnt++;
 
-       mlog(4," WC_rem hd %d tl %d, qpn 0x%x\n", m_qp->wc_hd_rem, m_qp->wc_tl_rem, ib_qp->qp_num);
+       mlog(4," WC_rem hd %d tl %d, m_qp %p qpn 0x%x, m_cm %p\n",
+               m_qp, m_qp->wc_hd_rem, m_qp->wc_tl_rem,
+               ib_qp->qp_num, m_qp->cm);
        return 0;
 }
 

diff --git a/dapl/svc/mpxyd.c b/dapl/svc/mpxyd.c
index 70ffb66f979866d0d76f642c2aa4c61f11ab3157..8385cccbdbe34c64e0d5436082fd4de8f49c83c4 100644 (file)
--- a/dapl/svc/mpxyd.c
+++ b/dapl/svc/mpxyd.c
@@ -315,7 +315,7 @@ void mpxy_destroy_smd(mcm_scif_dev_t *smd)
        while (m_cm) {
                next_cm = get_next_entry(&m_cm->entry, &smd->llist);
                mpxy_unlock(&smd->llock);
-               mcm_dqlisten(smd, m_cm); /* dequeue and free */
+               mcm_dqlisten_free(smd, m_cm); /* dequeue and free */
                mpxy_lock(&smd->llock);
                m_cm = next_cm;
        }
@@ -329,7 +329,7 @@ void mpxy_destroy_smd(mcm_scif_dev_t *smd)
        while (m_cm) {
                next_cm = get_next_entry(&m_cm->entry, &smd->clist);
                mpxy_unlock(&smd->clock);
-               mcm_dqconn(smd, m_cm); /* dequeue and free */
+               mcm_dqconn_free(smd, m_cm); /* dequeue and free */
                mpxy_lock(&smd->clock);
                m_cm = next_cm;
        }

diff --git a/dapl/svc/mpxyd.h b/dapl/svc/mpxyd.h
index 14dc09983bd450e386974088cf59c93bdbdc26c5..0095e5d5f8b6fbc6164634147cdb53f53b85e729 100644 (file)
--- a/dapl/svc/mpxyd.h
+++ b/dapl/svc/mpxyd.h
@@ -514,9 +514,9 @@ void mcm_check_timers(mcm_scif_dev_t *smd, int *timer);
 void mcm_ib_recv(mcm_ib_dev_t *md);
 int mcm_ib_async_event(struct mcm_ib_dev *md);
 void mcm_qlisten(mcm_scif_dev_t *smd, mcm_cm_t *cm);
-void mcm_dqlisten(mcm_scif_dev_t *smd, mcm_cm_t *cm);
+void mcm_dqlisten_free(mcm_scif_dev_t *smd, mcm_cm_t *cm);
 void mcm_qconn(mcm_scif_dev_t *smd, mcm_cm_t *cm);
-void mcm_dqconn(mcm_scif_dev_t *smd, mcm_cm_t *cm);
+void mcm_dqconn_free(mcm_scif_dev_t *smd, mcm_cm_t *cm);
 int mcm_modify_qp(struct ibv_qp *qp_handle, enum ibv_qp_state qp_state,
                  uint32_t qpn, uint16_t lid, union ibv_gid *gid);
 void mcm_flush_qp(struct mcm_qp *m_qp);
@@ -703,6 +703,7 @@ static inline int scif_send_msg(scif_epd_t ep, void *msg, int snd_len)
        int flags = 0; /* !SCIF_SEND_BLOCK */
 
        while (len) {
+               errno = 0;
                if (len == snd_len) mlog(8, "  scif_send - ep %d, msg %p len=%d \n", ep, msg+off, len);
                ret = scif_send(ep, msg+off, len, flags);
                if (len == snd_len) mlog(8, "  scif_sent - ep %d, len=%d ret=%d %s\n", ep, len, ret, strerror(errno));