mcm: process QP create errors from mpxyd

add checking and reporting for QP errors back to mix client
on the MIC adapter

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/openib_common/qp.c b/dapl/openib_common/qp.c
index 9e12d812bef4df7ab15e4480eeff8817d9ab9c5c..9286fdde6d4290dd07fa2a31d79d5df850c095fa 100644 (file)
--- a/dapl/openib_common/qp.c
+++ b/dapl/openib_common/qp.c
@@ -234,7 +234,9 @@ dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,
                             qp_create.cap.max_send_wr, qp_create.cap.max_send_sge,
                             qp_create.cap.max_recv_wr, qp_create.cap.max_recv_sge);
 
-               dapli_mix_qp_create(ep_ptr->qp_handle, &qp_create, req_cq, rcv_cq);
+               ret = dapli_mix_qp_create(ep_ptr->qp_handle, &qp_create, req_cq, rcv_cq);
+               if (ret)
+                       goto err;
        } else {
                /* NON-MIC: need QPt, in case of shadowed QP's from MIC's */
                qp_create.cap.max_recv_wr = 1;

diff --git a/dapl/openib_mcm/mix.c b/dapl/openib_mcm/mix.c
index e8cb1737e9eb1465beaa40e02a1d503bb373d6e9..ea79ddb3803b8d22f503acb49b6287f4bb0a0c6c 100644 (file)
--- a/dapl/openib_mcm/mix.c
+++ b/dapl/openib_mcm/mix.c
@@ -365,6 +365,7 @@ int dapli_mix_qp_create(ib_qp_handle_t m_qp, struct ibv_qp_init_attr *attr,
        ret = scif_send(mix_ep, &msg, len, SCIF_SEND_BLOCK);
        if (ret != len) {
                dapl_log(1, " ERR: send on %d, ret %d, exp %d, error %s\n", mix_ep, ret, len, strerror(errno));
+               return EFAULT;
        }
        dapl_log(DAPL_DBG_TYPE_EXTENSION," Sent %s request on SCIF EP\n", mix_op_str(msg.hdr.op));
 
@@ -372,13 +373,16 @@ int dapli_mix_qp_create(ib_qp_handle_t m_qp, struct ibv_qp_init_attr *attr,
        ret = scif_recv(mix_ep, &msg, len, SCIF_RECV_BLOCK);
        if (ret != len) {
                dapl_log(1, " ERR: rcv on new_ep %d, ret %d, exp %d, error %s\n", mix_ep, ret, len, strerror(errno));
-               return -1;
+               return EFAULT;
        }
        if (msg.hdr.ver != DAT_MIX_VER || msg.hdr.op != MIX_QP_CREATE ||
            msg.hdr.flags != MIX_OP_RSP || msg.hdr.status != MIX_SUCCESS) {
                dapl_log(1, " MIX msg ver %d, op %d, flags %d, or stat %d ERR \n",
                         msg.hdr.ver, msg.hdr.op, msg.hdr.flags, msg.hdr.status);
-               return -1;
+               if (msg.hdr.status)
+                       return msg.hdr.status;
+               else
+                       return EINVAL;
        }
 
        /* save QP_t id, ctx, and proxy buffer and wr pools. used on post_writes */
@@ -946,7 +950,7 @@ int dapli_mix_cm_event_in(ib_hca_transport_t *tp, scif_epd_t scif_ep, dat_mix_cm
        /* Find the CM and EP for event processing */
        cm = dapli_mix_get_cm(tp, pmsg->cm_ctx);
        if (!cm) {
-               dapl_log(DAPL_DBG_TYPE_ERR, " ERR: mcm_get_cm, ctx %p, not found\n", pmsg->cm_ctx);
+               dapl_log(DAPL_DBG_TYPE_CM_WARN, " mcm_get_cm, ctx %p, not found\n", pmsg->cm_ctx);
                return -1;
        }
 

diff --git a/dapl/svc/mpxyd.c b/dapl/svc/mpxyd.c
index 1305bde888b99614f2a3d8fdf6688d32c41621a5..39888e7a4a0daa723f70f02fe50f8c44f92816d2 100644 (file)
--- a/dapl/svc/mpxyd.c
+++ b/dapl/svc/mpxyd.c
@@ -2003,11 +2003,12 @@ static int m_qp_create(mcm_scif_dev_t *smd,
        *new_mqp = m_qp;
        return 0;
 err:
+       ret = errno;
        destroy_mbuf_pool(m_qp);
        destroy_wrbuf_pool(m_qp);
        if (m_qp)
                free(m_qp);
-       return -1;
+       return ret;
 }
 
 /* create new proxy QP */
@@ -2022,7 +2023,7 @@ static int mix_qp_create(mcm_scif_dev_t *smd, dat_mix_qp_t *pmsg)
        ret = scif_recv(smd->scif_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
        if (ret != len) {
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
-               return ret;
+               return -1;
        }
 
        mlog(1, " QP_r - qpn 0x%x, ctx %p, rq %d,%d sq %d,%d rcq_id %d\n",
@@ -2040,11 +2041,16 @@ static int mix_qp_create(mcm_scif_dev_t *smd, dat_mix_qp_t *pmsg)
        qp_create.cap.max_recv_sge = pmsg->qp_t.max_recv_sge;
        qp_create.cap.max_send_wr = pmsg->qp_t.max_send_wr * (mix_max_msg_mb*1024*1024/mix_buffer_sg);
        qp_create.cap.max_send_sge = pmsg->qp_t.max_send_sge;
-       qp_create.cap.max_inline_data = pmsg->qp_t.max_inline_data;
+       qp_create.cap.max_inline_data = 0; /* better bandwidth without inline */
        qp_create.qp_type = IBV_QPT_RC;
 
-       if (m_qp_create(smd, &qp_create, pmsg->qp_t.scq_id, &new_mqp))
-               goto err;
+       mlog(1, " QP_t - max_wr %d adjusted for segmentation\n",
+               pmsg->qp_t.max_send_wr, pmsg->qp_t.max_send_sge,
+               pmsg->qp_t.max_inline_data);
+
+       pmsg->hdr.status = m_qp_create(smd, &qp_create, pmsg->qp_t.scq_id, &new_mqp);
+       if (pmsg->hdr.status)
+               goto resp;
 
        /* return QPt info to MIC client, insert on QP list */
        memcpy(&pmsg->qp_t, &new_mqp->qp_t, sizeof(dat_mix_qp_attr_t));
@@ -2064,10 +2070,6 @@ static int mix_qp_create(mcm_scif_dev_t *smd, dat_mix_qp_t *pmsg)
                pmsg->qp_t.max_send_sge, pmsg->qp_t.max_recv_wr,
                pmsg->qp_t.max_recv_sge);
 
-       goto resp;
-err:
-       mlog(0, " ERR: %s\n", strerror(errno));
-       pmsg->hdr.status = MIX_EINVAL;
 resp:
        /* send back response */
        pmsg->hdr.flags = MIX_OP_RSP;