mpxyd,mcm: changes for backward compatibility with older v4 MIC clients

Allow mpxyd service to run with older MIC clients that support only proxy-out
and not proxy-in capabilities. Define minimal and compatible versions and
sync to MIC client during device open.

Create and use dat_mcm_msg_compat, dat_mix_mr_compat, and dat_mix_cm_compat
messages and operations with older v4 clients.

Move current MIX command version to v5.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/openib_mcm/cm.c b/dapl/openib_mcm/cm.c
index 59e9012a1334ad844c8f357f4fdadd69f37b4d0b..35c82c49bf4f6c37a3a7950c999df678e7952a62 100644 (file)
--- a/dapl/openib_mcm/cm.c
+++ b/dapl/openib_mcm/cm.c
@@ -630,12 +630,13 @@ static int mcm_send(ib_hca_transport_t *tp, dat_mcm_msg_t *msg, DAT_PVOID p_data
                goto bail;
        }
 
-       len = (sizeof(*msg) - DAT_MCM_PDATA_SIZE);
+       len = sizeof(dat_mcm_msg_t);
        dapl_os_memcpy(smsg, msg, len);
        if (p_size) {
                smsg->p_size = ntohs(p_size);
                dapl_os_memcpy(&smsg->p_data, p_data, p_size);
-       }
+       } else
+               smsg->p_size = 0;
 
        wr.next = NULL;
         wr.sg_list = &sge;
@@ -646,7 +647,7 @@ static int mcm_send(ib_hca_transport_t *tp, dat_mcm_msg_t *msg, DAT_PVOID p_data
        if (len <= tp->max_inline_send)
                wr.send_flags |= IBV_SEND_INLINE; 
 
-        sge.length = len + p_size;
+        sge.length = len;
         sge.lkey = tp->mr_sbuf->lkey;
         sge.addr = (uintptr_t)smsg;
 
@@ -1178,7 +1179,7 @@ void mcm_connect_rtu(dp_ib_cm_handle_t cm, dat_mcm_msg_t *msg)
                if (MXS_EP(&cm->msg.daddr1)) {
                        /* save PI WR info, create local WC_q, send back WC info */
                        mcm_ntoh_wrc(&ep->qp_handle->wrc_rem, (mcm_wrc_info_t*)cm->msg.p_proxy);
-                       mcm_create_wc_q(ep->qp_handle, MCM_WRC_QLEN);
+                       mcm_create_wc_q(ep->qp_handle, ep->qp_handle->wrc_rem.wr_end + 1);
                        mcm_hton_wrc((mcm_wrc_info_t*)cm->msg.p_proxy, &ep->qp_handle->wrc);
                        ep->qp_handle->ep_map = cm->msg.daddr1.ep_map;
 
@@ -1186,8 +1187,7 @@ void mcm_connect_rtu(dp_ib_cm_handle_t cm, dat_mcm_msg_t *msg)
                        if (mcm_post_rcv_wc(ep->qp_handle, MCM_WRC_QLEN))
                                goto bail;
 
-                       dapl_log(DAPL_DBG_TYPE_CM,
-                                "CONN_RTU: WR_rem %p sz %d, WC %p sz %d\n",
+                       dapl_log(DAPL_DBG_TYPE_CM, "CONN_RTU: WR_rem %p sz %d, WC %p sz %d\n",
                                 ep->qp_handle->wrc_rem.wr_addr,
                                 ep->qp_handle->wrc_rem.wr_end+1,
                                 ep->qp_handle->wrc.wc_addr,
@@ -1498,7 +1498,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
                if (MXS_EP(&cm->msg.daddr1)) {
                        /* save PI WR info, create local WC_q, send back WC info */
                        mcm_ntoh_wrc(&ep->qp_handle->wrc_rem, (mcm_wrc_info_t*)cm->msg.p_proxy);
-                       mcm_create_wc_q(ep->qp_handle, MCM_WRC_QLEN);
+                       mcm_create_wc_q(ep->qp_handle, ep->qp_handle->wrc_rem.wr_end + 1);
                        mcm_hton_wrc((mcm_wrc_info_t*)cm->msg.p_proxy, &ep->qp_handle->wrc);
                        ep->qp_handle->ep_map = cm->msg.daddr1.ep_map;
 
@@ -1507,10 +1507,12 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
                                        goto bail;
 
                        dapl_log(DAPL_DBG_TYPE_CM,
-                                "ACCEPT_USR: WR_rem %p sz %d, WC %p sz %d\n",
+                                "ACCEPT_USR: WR_rem %p rkey %x sz %d, WC %p rkey %x sz %d\n",
                                 ep->qp_handle->wrc_rem.wr_addr,
+                                ep->qp_handle->wrc_rem.wr_rkey,
                                 ep->qp_handle->wrc_rem.wr_end+1,
                                 ep->qp_handle->wrc.wc_addr,
+                                ep->qp_handle->wrc.wc_rkey,
                                 ep->qp_handle->wrc.wc_end+1);
                }
        }

diff --git a/dapl/openib_mcm/mix.c b/dapl/openib_mcm/mix.c
index 4d27b537f5dda296547e4b695d83658fb9d033bf..06656dca4edde829fa790eb8308a3dc01cb269b1 100644 (file)
--- a/dapl/openib_mcm/mix.c
+++ b/dapl/openib_mcm/mix.c
@@ -518,8 +518,8 @@ int dapli_mix_qp_free(ib_qp_handle_t m_qp)
        }
        if (msg.ver != DAT_MIX_VER || msg.op != MIX_QP_FREE ||
            msg.flags != MIX_OP_RSP || msg.status != MIX_SUCCESS) {
-               dapl_log(1, " MIX msg ver %d, op %d, flags %d, or stat %d ERR \n",
-                        msg.ver, msg.op, msg.flags, msg.status);
+               dapl_log(1, " MIX_QP_FREE ERR: ver %d, op %d, flags %d, or stat %d len %d\n",
+                        msg.ver, msg.op, msg.flags, msg.status, ret);
                return -1;
        }
        dapl_log(DAPL_DBG_TYPE_EXTENSION," received reply on SCIF EP\n");
@@ -551,6 +551,8 @@ int dapli_mix_cq_create(ib_cq_handle_t m_cq, int cq_len)
                         strerror(errno));
                return -1;
        }
+       dapl_log(DAPL_DBG_TYPE_EXTENSION," Sent %s request on SCIF EP\n",
+                mix_op_str(msg.hdr.op));
 
        /* wait for response */
        ret = scif_recv(mix_ep, &msg, len, SCIF_RECV_BLOCK);
@@ -595,6 +597,8 @@ int dapli_mix_cq_free(ib_cq_handle_t m_cq)
                dapl_log(1, " ERR: %s send on %d, ret %d, exp %d, error %s\n",
                         mix_op_str(msg.op), mix_ep, ret, len, strerror(errno));
        }
+       dapl_log(DAPL_DBG_TYPE_EXTENSION," Sent %s request on SCIF EP\n",
+                mix_op_str(msg.op));
 
        /* response */
        ret = scif_recv(mix_ep, &msg, len, SCIF_RECV_BLOCK);
@@ -605,10 +609,12 @@ int dapli_mix_cq_free(ib_cq_handle_t m_cq)
        }
        if (msg.ver != DAT_MIX_VER || msg.op != MIX_CQ_FREE ||
            msg.flags != MIX_OP_RSP || msg.status != MIX_SUCCESS) {
-               dapl_log(1, " MIX msg ver %d, op %d, flags %d, or stat %d ERR \n",
-                        msg.ver, msg.op, msg.flags, msg.status);
+               dapl_log(1, " MIX_CQ_FREE ERR: ver %d, op %d, flags %d, or stat %d ln %d\n",
+                        msg.ver, msg.op, msg.flags, msg.status, ret);
                return -1;
        }
+       dapl_log(DAPL_DBG_TYPE_EXTENSION,
+                " MIX_CQ_FREE: reply, proxy cq_id 0x%x\n", m_cq->cq_id);
        return 0;
 }
 
@@ -940,7 +946,7 @@ int dapli_mix_cm_rtu_out(dp_ib_cm_handle_t m_cm)
        msg.cm_ctx = (uint64_t)m_cm;
 
        dapl_log(DAPL_DBG_TYPE_EXTENSION," RTU -> id 0x%x dport 0x%x, dqpn 0x%x dlid 0x%x\n",
-                msg.cm_id, ntohs(msg.msg.dport), ntohl(msg.msg.dqpn), ntohs(msg.msg.daddr1.lid) );
+                msg.cm_id, ntohs(msg.msg.dport), ntohl(msg.msg.dqpn), ntohs(msg.msg.daddr1.lid));
 
        len = sizeof(dat_mix_cm_t);
        dapl_os_lock(&m_cm->tp->lock);

diff --git a/dapl/svc/mcm.c b/dapl/svc/mcm.c
index e6207e798118e4d678120ac6b374810c37e77586..3377e47711d256c59679887dad3c3f30746c7fad 100644 (file)
--- a/dapl/svc/mcm.c
+++ b/dapl/svc/mcm.c
@@ -174,8 +174,8 @@ int mcm_init_cm_service(mcm_ib_dev_t *md)
                     strerror(errno));
                return -1;
        }
-       /* endpoint mapping hint for MIC to HCA communication */
-       if (md->numa_node == md->mc->numa_node)
+       /* endpoint mapping hint for MIC to HCA communication, backward compatible */
+       if ((md->numa_node == md->mc->numa_node) || (md->mc->ver == MIX_COMP))
                md->addr.ep_map = MIC_SSOCK_DEV;
        else
                md->addr.ep_map = MIC_XSOCK_DEV;
@@ -270,11 +270,11 @@ int mcm_init_cm_service(mcm_ib_dev_t *md)
        /* save qp_num as part of ia_address, network order */
        md->addr.qpn = htonl(md->qp->qp_num);
 
-       mlog(0, " IB LID 0x%x PORT %d GID %s QPN 0x%x: mic%d -> %s - %s socket\n",
+       mlog(0, " IB LID 0x%x PORT %d GID %s QPN 0x%x: mic%d -> %s - %s, mic_ver %d\n",
                ntohs(md->addr.lid), md->port,
                inet_ntop(AF_INET6, md->addr.gid, gid_str, sizeof(gid_str)),
                md->qp->qp_num, md->mc->scif_id - 1, md->ibdev->name,
-               md->addr.ep_map == MIC_SSOCK_DEV ? "local":"cross");
+               md->addr.ep_map == MIC_SSOCK_DEV ? "MSS":"MXS", md->mc->ver);
 
        return 0;
 bail:
@@ -758,12 +758,13 @@ static int mcm_send(mcm_ib_dev_t *md, dat_mcm_msg_t *msg, DAT_PVOID p_data, DAT_
                goto bail;
        }
 
-       len = (sizeof(dat_mcm_msg_t) - DAT_MCM_PDATA_SIZE);
+       len = sizeof(dat_mcm_msg_t);
        memcpy(smsg, msg, len);
        if (p_size) {
                smsg->p_size = ntohs(p_size);
                memcpy(&smsg->p_data, p_data, p_size);
-       }
+       } else
+               smsg->p_size = 0;
 
        wr.next = NULL;
         wr.sg_list = &sge;
@@ -773,7 +774,7 @@ static int mcm_send(mcm_ib_dev_t *md, dat_mcm_msg_t *msg, DAT_PVOID p_data, DAT_
        wr.send_flags = (wr.wr_id % md->signal) ? 0 : IBV_SEND_SIGNALED;
        wr.send_flags |= IBV_SEND_INLINE;
 
-        sge.length = len + p_size;
+        sge.length = len;
         sge.lkey = md->mr_sbuf->lkey;
         sge.addr = (uintptr_t)smsg;
 

diff --git a/dapl/svc/mix.c b/dapl/svc/mix.c
index db59f9b066fe913299ffc0644a0e9c93d605290d..8d19bfc3ac97b2cdb8cee36fff8b274166026c4b 100644 (file)
--- a/dapl/svc/mix.c
+++ b/dapl/svc/mix.c
@@ -166,7 +166,7 @@ void mix_scif_accept(scif_epd_t listen_ep)
 
        msg.hdr.flags = MIX_OP_RSP;
 
-       if (msg.hdr.ver != DAT_MIX_VER || msg.hdr.op != MIX_IA_OPEN) {
+       if (msg.hdr.ver < MIX_MIN || msg.hdr.ver > MIX_MAX || msg.hdr.op != MIX_IA_OPEN) {
                mlog(0, " ERR: MIC client incompatible with MPXYD (exp %d,rcvd %d) or OP (exp %d,rcvd %d)\n",
                        DAT_MIX_VER, msg.hdr.ver, msg.hdr.op, MIX_IA_OPEN);
                msg.hdr.ver = DAT_MIX_VER;
@@ -857,6 +857,16 @@ resp:
        return (scif_send_msg(smd->scif_op_ep, (void*)pmsg, len));
 }
 
+typedef struct dat_mix_mr_v4
+{
+       dat_mix_hdr_t           hdr;
+       uint32_t                mr_id;
+       uint32_t                len;
+       uint64_t                off;
+       uint64_t                ctx;
+
+} dat_mix_mr_v4_t;
+
 /* MIX_MR_CREATE: new proxy mr, insert on mr_list */
 static int mix_mr_create(mcm_scif_dev_t *smd, dat_mix_mr_t *pmsg)
 {
@@ -864,7 +874,12 @@ static int mix_mr_create(mcm_scif_dev_t *smd, dat_mix_mr_t *pmsg)
        struct mcm_mr *m_mr = NULL;
 
        /* hdr already read, get operation data */
-       len = sizeof(dat_mix_mr_t) - sizeof(dat_mix_hdr_t);
+
+       if (smd->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_mr_compat_t) - sizeof(dat_mix_hdr_t);
+       else
+               len = sizeof(dat_mix_mr_t) - sizeof(dat_mix_hdr_t);
+
        ret = scif_recv(smd->scif_op_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
        if (ret != len) {
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
@@ -896,7 +911,11 @@ static int mix_mr_create(mcm_scif_dev_t *smd, dat_mix_mr_t *pmsg)
 resp:
        /* send back response */
        pmsg->hdr.flags = MIX_OP_RSP;
-       len = sizeof(dat_mix_mr_t);
+       if (smd->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_mr_compat_t);
+       else
+               len = sizeof(dat_mix_mr_t);
+
        return (scif_send_msg(smd->scif_op_ep, (void*)pmsg, len));
 }
 
@@ -957,7 +976,7 @@ void mix_dto_event(struct mcm_cq *m_cq, struct dat_mix_wc *wc, int nc)
        int i;
 
        /* send DTO events to MIC client */
-       msg.hdr.ver = DAT_MIX_VER;
+       msg.hdr.ver = m_cq->smd->md->mc->ver;
        msg.hdr.op = MIX_DTO_EVENT;
        msg.hdr.flags = MIX_OP_REQ;
        msg.cq_id = m_cq->cq_id;
@@ -999,7 +1018,7 @@ void mix_cm_event(mcm_cm_t *m_cm, uint32_t event)
        int len;
 
        /* send event to MIC client */
-       msg.hdr.ver = DAT_MIX_VER;
+       msg.hdr.ver = m_cm->md->mc->ver;
        msg.hdr.op = MIX_CM_EVENT;
        msg.hdr.flags = MIX_OP_REQ;
        msg.cm_id = m_cm->cm_id;
@@ -1031,11 +1050,15 @@ static int mix_cm_req_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        struct mcm_qp *m_qp = NULL;
        struct mcm_cm *m_cm = NULL;
 
-       /* hdr already read, get operation data */
-       len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+       /* hdr already read, get operation data, support compat mode */
+       if (smd->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t) - sizeof(dat_mix_hdr_t);
+       else
+               len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+
        ret = scif_recv(scif_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
        if (ret != len) {
-               mlog(0, " ERR: ret %d, exp %d\n", ret, len);
+               mlog(0, " ERR: ret %d, exp %d ver %d\n", ret, len, smd->md->mc->ver);
                return ret;
        }
 
@@ -1111,7 +1134,12 @@ resp:
        /* send back response */
        pmsg->hdr.flags = MIX_OP_RSP;
 
-       len = sizeof(dat_mix_cm_t);
+       /* support compat mode */
+       if (m_cm->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t);
+       else
+               len = sizeof(dat_mix_cm_t);
+
        return (scif_send_msg(smd->scif_op_ep, (void*)pmsg, len));
 }
 
@@ -1121,8 +1149,12 @@ static int mix_cm_disc_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t s
        int len, ret;
        struct mcm_cm *m_cm;
 
-       /* hdr already read, get operation data */
-       len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+       /* hdr already read, get operation data, support compat mode */
+       if (smd->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t) - sizeof(dat_mix_hdr_t);
+       else
+               len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+
        ret = scif_recv(scif_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
        if (ret != len) {
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
@@ -1149,8 +1181,12 @@ static int mix_cm_rtu_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        int len, ret;
        struct mcm_cm *m_cm;
 
-       /* hdr already read, get operation data */
-       len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+       /* hdr already read, get operation data, support compat mode */
+       if (smd->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t) - sizeof(dat_mix_hdr_t);
+       else
+               len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+
        ret = scif_recv(scif_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
        if (ret != len) {
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
@@ -1166,12 +1202,11 @@ static int mix_cm_rtu_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
                return -1;
        }
 
-       mlog(2," CONNECTED: QPt 0x%x QPr 0x%x -> dport 0x%x, dqpn 0x%x dlid 0x%x psize %d %Lx\n",
+       mlog(2," CONNECTED: QPt 0x%x QPr 0x%x -> dport 0x%x, dqpn 0x%x dlid 0x%x %Lx\n",
                 m_cm->m_qp?m_cm->m_qp->qp_attr2.qp_num:0,
                 m_cm->m_qp?m_cm->m_qp->qp_attr1.qp_num:0,
                 ntohs(m_cm->msg.dport), ntohl(m_cm->msg.dqpn),
-                ntohs(m_cm->msg.daddr1.lid), ntohs(m_cm->msg.p_size),
-                ntohll(m_cm->msg.sys_guid));
+                ntohs(m_cm->msg.daddr1.lid), ntohll(m_cm->msg.sys_guid));
 
        /* send RTU on wire */
        mcm_cm_rtu_out(m_cm);
@@ -1197,7 +1232,7 @@ int mix_cm_rej_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
                MCNTR(m_cm->md, MCM_CM_REJ_IN);
                msg.hdr.op = MIX_CM_REJECT;
        }
-       msg.hdr.ver = DAT_MIX_VER;
+       msg.hdr.ver = m_cm->md->mc->ver;
        msg.hdr.flags = MIX_OP_REQ;
        msg.cm_id = m_cm->cm_id;
        msg.cm_ctx = m_cm->cm_ctx;
@@ -1219,7 +1254,12 @@ int mix_cm_rej_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
 
        /* clean up proxy QP, CQ resources here ??? */
 
-       len = sizeof(dat_mix_cm_t);
+       /* support compat mode */
+       if (m_cm->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t);
+       else
+               len = sizeof(dat_mix_cm_t);
+
        mpxy_lock(&m_cm->smd->evlock);
        if (scif_send_msg(m_cm->smd->scif_ev_ep, (void*)&msg, len)) {
                mpxy_unlock(&m_cm->smd->evlock);
@@ -1250,7 +1290,7 @@ int mix_cm_rep_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
             pkt_len, ntohs(pkt->p_size), ntohll(pkt->sys_guid));
 
        /* Forward, as is, conn_reply message to MIC client, with remote QP info */
-       msg.hdr.ver = DAT_MIX_VER;
+       msg.hdr.ver = m_cm->md->mc->ver;
        msg.hdr.flags = MIX_OP_REQ;
        msg.hdr.op = MIX_CM_REP;
        msg.cm_id = m_cm->cm_id;
@@ -1268,10 +1308,21 @@ int mix_cm_rep_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
        memcpy(&m_cm->msg.daddr2, &pkt->saddr2, sizeof(dat_mcm_addr_t));
        mcm_ntoh_wrc(&m_cm->m_qp->wrc_rem, (mcm_wrc_info_t *)m_cm->msg.p_proxy); /* peer RI WRC info */
 
-       mlog(2, " WRC_rem: m_qp %p - addr 0x%Lx rkey 0x%x len %d, sz %d end %d\n",
-            m_cm->m_qp, m_cm->m_qp->wrc_rem.wc_addr, m_cm->m_qp->wrc_rem.wc_rkey,
-            m_cm->m_qp->wrc_rem.wc_len, m_cm->m_qp->wrc_rem.wc_sz,
-            m_cm->m_qp->wrc_rem.wc_end);
+       mlog(2, " WRC: m_qp %p - WR 0x%Lx rkey 0x%x ln %d, sz %d end %d"
+               " WC 0x%Lx rkey 0x%x ln %d, sz %d end %d\n",
+            m_cm->m_qp, m_cm->m_qp->wrc.wr_addr, m_cm->m_qp->wrc.wr_rkey,
+            m_cm->m_qp->wrc.wr_len, m_cm->m_qp->wrc.wr_sz,
+            m_cm->m_qp->wrc.wr_end, m_cm->m_qp->wrc.wc_addr,
+            m_cm->m_qp->wrc.wc_rkey, m_cm->m_qp->wrc.wc_len,
+            m_cm->m_qp->wrc.wc_sz, m_cm->m_qp->wrc.wc_end);
+
+       mlog(2, " WRC_rem: m_qp %p - WR 0x%Lx rkey 0x%x ln %d, sz %d end %d"
+               " WC 0x%Lx rkey 0x%x ln %d, sz %d end %d\n",
+            m_cm->m_qp, m_cm->m_qp->wrc_rem.wr_addr, m_cm->m_qp->wrc_rem.wr_rkey,
+            m_cm->m_qp->wrc_rem.wr_len, m_cm->m_qp->wrc_rem.wr_sz,
+            m_cm->m_qp->wrc_rem.wr_end, m_cm->m_qp->wrc_rem.wc_addr,
+            m_cm->m_qp->wrc_rem.wc_rkey, m_cm->m_qp->wrc_rem.wc_len,
+            m_cm->m_qp->wrc_rem.wc_sz, m_cm->m_qp->wrc_rem.wc_end);
 
        /* MXS <- MSS or HOST, fabric: TX: QP2->QP1 direct, RX: QP1<-QP2 proxy */
        if ((MXS_EP(&m_cm->md->addr) && !MXS_EP(&m_cm->msg.daddr1)) &&
@@ -1355,7 +1406,12 @@ int mix_cm_rep_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
                        goto err;
        }
 
-       len = sizeof(dat_mix_cm_t);
+       /* support compat mode */
+       if (m_cm->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t);
+       else
+               len = sizeof(dat_mix_cm_t);
+
        mpxy_lock(&m_cm->smd->evlock);
        if (scif_send_msg(m_cm->smd->scif_ev_ep, (void*)&msg, len)) {
                mpxy_unlock(&m_cm->smd->evlock);
@@ -1390,7 +1446,7 @@ int mix_cm_req_in(mcm_cm_t *cm, dat_mcm_msg_t *pkt, int pkt_len)
 
        mcm_pr_addrs(2, pkt, acm->state, 1);
 
-       msg.hdr.ver = DAT_MIX_VER;
+       msg.hdr.ver = cm->md->mc->ver;
        msg.hdr.flags = MIX_OP_REQ;
        msg.hdr.op = MIX_CM_REQ;
        msg.hdr.status = MIX_SUCCESS;
@@ -1427,8 +1483,14 @@ int mix_cm_req_in(mcm_cm_t *cm, dat_mcm_msg_t *pkt, int pkt_len)
        if (pkt->p_size)
                memcpy(acm->msg.p_data, pkt->p_data, ntohs(pkt->p_size));
 
-       /* forward reformated CM message info to MIX client */
-       memcpy(&msg.msg, &acm->msg, sizeof(dat_mcm_msg_t));
+       /* forward reformated CM message info to MIX client, support compat mode */
+       if (cm->md->mc->ver == MIX_COMP) {
+               len = sizeof(dat_mix_cm_compat_t);
+               memcpy(&msg.msg, &acm->msg, sizeof(dat_mcm_msg_compat_t));
+       } else {
+               len = sizeof(dat_mix_cm_t);
+               memcpy(&msg.msg, &acm->msg, sizeof(dat_mcm_msg_t));
+       }
 
        acm->state = MCM_ACCEPTING;
        mcm_qconn(acm->smd, acm);
@@ -1436,7 +1498,6 @@ int mix_cm_req_in(mcm_cm_t *cm, dat_mcm_msg_t *pkt, int pkt_len)
        msg.cm_ctx = (uint64_t)acm;
        msg.sp_ctx = cm->sp_ctx;
 
-       len = sizeof(dat_mix_cm_t);
        mpxy_lock(&acm->smd->evlock);
        if (scif_send_msg(acm->smd->scif_ev_ep, (void*)&msg, len)) {
                mpxy_unlock(&acm->smd->evlock);
@@ -1472,14 +1533,19 @@ int mix_cm_rtu_in(mcm_cm_t *m_cm, dat_mcm_msg_t *pkt, int pkt_len)
        }
 
        /* Forward, as is, conn_reply message to MIC client, with remote QP info */
-       msg.hdr.ver = DAT_MIX_VER;
+       msg.hdr.ver = m_cm->md->mc->ver;
        msg.hdr.flags = MIX_OP_REQ;
        msg.hdr.op = MIX_CM_RTU;
        msg.cm_id = m_cm->cm_id;
        msg.cm_ctx = m_cm->cm_ctx;
        m_cm->msg.sys_guid = pkt->sys_guid; /* save remote quid */
 
-       len = sizeof(dat_mix_cm_t);
+       /* support compat mode */
+       if (m_cm->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t);
+       else
+               len = sizeof(dat_mix_cm_t);
+
        mpxy_lock(&m_cm->smd->evlock);
        if (scif_send_msg(m_cm->smd->scif_ev_ep, (void*)&msg, len)) {
                mpxy_unlock(&m_cm->smd->evlock);
@@ -1500,8 +1566,12 @@ static int mix_cm_rep_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        uint32_t dqpn = 0, dqpn2 = 0;
        uint16_t dlid = 0, dlid2 = 0;
 
-       /* hdr already read, get operation data */
-       len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+       /* hdr already read, get operation data, support compat mode */
+       if (smd->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t) - sizeof(dat_mix_hdr_t);
+       else
+               len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+
        ret = scif_recv(scif_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
        if (ret != len) {
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
@@ -1523,7 +1593,11 @@ static int mix_cm_rep_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        m_cm->cm_ctx = pmsg->sp_ctx;
        m_cm->p_size = ntohs(pmsg->msg.p_size);
        memcpy(&m_cm->p_data, &pmsg->msg.p_data, m_cm->p_size);
-       memcpy(&m_cm->msg, &pmsg->msg, sizeof(dat_mcm_msg_t));
+
+       if (smd->md->mc->ver == MIX_COMP)
+               memcpy(&m_cm->msg, &pmsg->msg, sizeof(dat_mcm_msg_compat_t));
+       else
+               memcpy(&m_cm->msg, &pmsg->msg, sizeof(dat_mcm_msg_t));
 
        /* Attach the QP for this CR */
        m_cm->m_qp = mix_get_qp(smd, pmsg->qp_id);
@@ -1535,13 +1609,22 @@ static int mix_cm_rep_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        m_cm->ref_cnt++; /* Passive: QP ref */
        m_cm->m_qp->cm = m_cm;
        mcm_ntoh_wrc(&m_cm->m_qp->wrc_rem, (mcm_wrc_info_t *)m_cm->msg.p_proxy); /* save peer PI WRC info */
-       mlog(2, " WRC_rem: m_qp %p - addr 0x%Lx rkey 0x%x len %d, sz %d end %d\n",
-            m_cm->m_qp, m_cm->m_qp->wrc_rem.wc_addr, m_cm->m_qp->wrc_rem.wc_rkey,
-            m_cm->m_qp->wrc_rem.wc_len, m_cm->m_qp->wrc_rem.wc_sz,
-            m_cm->m_qp->wrc_rem.wc_end);
 
-       mlog(8, " loc_guid %Lx, rem_guid %Lx\n",
-               ntohll(system_guid), ntohll(m_cm->msg.sys_guid));
+       mlog(2, " WRC: m_qp %p - WR 0x%Lx rkey 0x%x ln %d, sz %d end %d"
+               " WC 0x%Lx rkey 0x%x ln %d, sz %d end %d\n",
+            m_cm->m_qp, m_cm->m_qp->wrc.wr_addr, m_cm->m_qp->wrc.wr_rkey,
+            m_cm->m_qp->wrc.wr_len, m_cm->m_qp->wrc.wr_sz,
+            m_cm->m_qp->wrc.wr_end, m_cm->m_qp->wrc.wc_addr,
+            m_cm->m_qp->wrc.wc_rkey, m_cm->m_qp->wrc.wc_len,
+            m_cm->m_qp->wrc.wc_sz, m_cm->m_qp->wrc.wc_end);
+
+       mlog(2, " WRC_rem: m_qp %p - WR 0x%Lx rkey 0x%x ln %d, sz %d end %d"
+               " WC 0x%Lx rkey 0x%x ln %d, sz %d end %d\n",
+            m_cm->m_qp, m_cm->m_qp->wrc_rem.wr_addr, m_cm->m_qp->wrc_rem.wr_rkey,
+            m_cm->m_qp->wrc_rem.wr_len, m_cm->m_qp->wrc_rem.wr_sz,
+            m_cm->m_qp->wrc_rem.wr_end, m_cm->m_qp->wrc_rem.wc_addr,
+            m_cm->m_qp->wrc_rem.wc_rkey, m_cm->m_qp->wrc_rem.wc_len,
+            m_cm->m_qp->wrc_rem.wc_sz, m_cm->m_qp->wrc_rem.wc_end);
 
        /* MXS -> MSS or HOST, remote: need QPr1, saddr1 on mpxyd */
        if ((MXS_EP(&m_cm->md->addr) && !MXS_EP(&m_cm->msg.daddr1)) &&
@@ -1704,8 +1787,12 @@ static int mix_cm_rej_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
        int len, ret;
        struct mcm_cm *m_cm;
 
-       /* hdr already read, get operation data */
-       len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+       /* hdr already read, get operation data, support compat mode */
+       if (smd->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t) - sizeof(dat_mix_hdr_t);
+       else
+               len = sizeof(dat_mix_cm_t) - sizeof(dat_mix_hdr_t);
+
        ret = scif_recv(scif_ep, ((char*)pmsg + sizeof(dat_mix_hdr_t)), len, SCIF_RECV_BLOCK);
        if (ret != len) {
                mlog(0, " ERR: ret %d, exp %d\n", ret, len);
@@ -1722,7 +1809,10 @@ static int mix_cm_rej_out(mcm_scif_dev_t *smd, dat_mix_cm_t *pmsg, scif_epd_t sc
                smd->entry.tid, pmsg->cm_id, (void*)pmsg->cm_ctx, pmsg->qp_id, m_cm);
 
        /* update CM message from MIX client, send back guid */
-       memcpy(&m_cm->msg, &pmsg->msg, sizeof(dat_mcm_msg_t));
+       if (smd->md->mc->ver == MIX_COMP)
+               memcpy(&m_cm->msg, &pmsg->msg, sizeof(dat_mcm_msg_compat_t));
+       else
+               memcpy(&m_cm->msg, &pmsg->msg, sizeof(dat_mcm_msg_t));
 
 #ifdef MPXYD_LOCAL_SUPPORT
        m_cm->msg.sys_guid = system_guid;
@@ -1743,12 +1833,17 @@ int mix_cm_disc_in(mcm_cm_t *m_cm)
        int len;
 
        /* send disconnect to MIC client */
-       msg.ver = DAT_MIX_VER;
+       msg.ver = m_cm->md->mc->ver;
        msg.flags = MIX_OP_REQ;
        msg.op = MIX_CM_DISC;
        msg.req_id = m_cm->cm_id;
 
-       len = sizeof(dat_mix_hdr_t);
+       /* support compat mode */
+       if (m_cm->md->mc->ver == MIX_COMP)
+               len = sizeof(dat_mix_cm_compat_t);
+       else
+               len = sizeof(dat_mix_cm_t);
+
        mpxy_lock(&m_cm->smd->evlock);
        if (scif_send_msg(m_cm->smd->scif_ev_ep, (void*)&msg, len)) {
                return -1;
@@ -1981,7 +2076,7 @@ int mix_scif_recv(mcm_scif_dev_t *smd, scif_epd_t scif_ep)
        len = sizeof(*phdr);
        phdr->ver = 0; phdr->op = 0;
        ret = scif_recv(scif_ep, phdr, len, SCIF_RECV_BLOCK);
-       if ((ret != len) || (phdr->ver != DAT_MIX_VER)) {
+       if ((ret != len) || (phdr->ver < MIX_MIN) || (phdr->ver > MIX_MAX)) {
                mlog(0,
                     " ERR: smd %p ep %d ret %d exp %d ver %d op %s flgs %d\n",
                     smd, scif_ep, ret, len, phdr->ver, mix_op_str(phdr->op),

diff --git a/dapl/svc/mpxyd.c b/dapl/svc/mpxyd.c
index 8385cccbdbe34c64e0d5436082fd4de8f49c83c4..2b83509f59733272392a1a6b0090754775de1668 100644 (file)
--- a/dapl/svc/mpxyd.c
+++ b/dapl/svc/mpxyd.c
@@ -634,6 +634,7 @@ mcm_scif_dev_t *mix_open_device(dat_mix_open_t *msg, scif_epd_t op_ep, scif_epd_
                char value[64];
                char path[64];
 
+               mc->ver = msg->hdr.ver;
                mc->scif_id = node;
                sprintf(path, "/sys/class/mic/mic%d/device", mc->scif_id - 1);
 

diff --git a/dapl/svc/mpxyd.h b/dapl/svc/mpxyd.h
index 0095e5d5f8b6fbc6164634147cdb53f53b85e729..49a1f18eaba8a7cf24b91d23e2d25a9145c20ab4 100644 (file)
--- a/dapl/svc/mpxyd.h
+++ b/dapl/svc/mpxyd.h
@@ -62,6 +62,9 @@
 #define MCM_IB_INLINE          160
 #define MIX_MAX_MSG_SIZE       (8*1024*1024)
 
+#define MIX_MIN  4             /* oldest version supported */
+#define MIX_COMP 4             /* compatibility version */
+#define MIX_MAX DAT_MIX_VER
 #define MCM_PROFILE 1
 #define MCM_PROFILE_DBG 0
 
@@ -355,6 +358,7 @@ typedef struct mcm_scif_dev {
 #define MCM_IB_MAX 8
 #define MCM_CLIENT_MAX 8
 typedef struct mcm_client {
+       uint16_t ver;
        uint16_t scif_id;
        int numa_node;
        int cpu_mask;

diff --git a/dat/include/dat2/dat_mic_extensions.h b/dat/include/dat2/dat_mic_extensions.h
index 15bfe2a5da837ab4dd0b8305a9dc1dc1263f4e08..a23b115294c9a9e44cc7a258460d5687ff863004 100755 (executable)
--- a/dat/include/dat2/dat_mic_extensions.h
+++ b/dat/include/dat2/dat_mic_extensions.h
@@ -217,7 +217,7 @@ typedef struct dat_mcm_addr
        uint8_t         qp_type;
 } __attribute__((packed)) dat_mcm_addr_t;
 
-/* MCM message extended, 256 bytes */
+/* MCM message extended after existing fields, 256 bytes */
 typedef struct dat_mcm_msg
 {
        uint16_t                ver;
@@ -236,12 +236,34 @@ typedef struct dat_mcm_msg
        dat_mcm_addr_t          saddr2; /* QPr local,  MIC  or MCM on non-MIC node or MPXY */
        dat_mcm_addr_t          daddr1; /* QPt remote, MPXY or MCM on non-MIC node */
        dat_mcm_addr_t          daddr2; /* QPr remote, MIC  or MCM on non-MIC node or MPXY */
-       uint64_t                sys_guid; /* system image guid */
-       uint8_t                 p_proxy[DAT_MCM_PROXY_DATA];
        uint8_t                 p_data[DAT_MCM_PDATA_SIZE];
+       uint8_t                 p_proxy[DAT_MCM_PROXY_DATA];
+       uint64_t                sys_guid; /* system image guid */
 
 } __attribute__((packed)) dat_mcm_msg_t;
 
+/* MCM message, 208 bytes */
+typedef struct dat_mcm_msg_compat
+{
+       uint16_t                ver;
+       uint16_t                op;
+       uint16_t                sport; /* src cm port */
+       uint16_t                dport; /* dst cm port */
+       uint32_t                sqpn;  /* src cm qpn */
+       uint32_t                dqpn;  /* dst cm qpn */
+       uint16_t                p_size;
+       uint32_t                s_id;  /* src pid */
+       uint32_t                d_id;  /* dst pid */
+       uint8_t                 rd_in; /* atomic_rd_in */
+       uint8_t                 resv[5];/* Shadow QP's, 2 connections */
+       dat_mcm_addr_t          saddr;  /* QPt local,  MPXY or MCM on non-MIC node */
+       dat_mcm_addr_t          saddr2; /* QPr local,  MIC  or MCM on non-MIC node */
+       dat_mcm_addr_t          daddr;  /* QPt remote, MPXY or MCM on non-MIC node */
+       dat_mcm_addr_t          daddr2; /* QPr remote, MIC  or MCM on non-MIC node */
+       uint8_t                 p_data[DAT_MCM_PDATA_SIZE];
+
+} __attribute__((packed)) dat_mcm_msg_compat_t;
+
 /***** MIC Indirect Exchange (MIX) protocol over SCIF ****/
 
 /* Revisions:
@@ -249,8 +271,9 @@ typedef struct dat_mcm_msg
  * v2 - Support 3 separate EP's per device (Operations/CM, unsolicited events, transmit)
  * v3 - reduce SGE from 7 to 4, add post_send inline support
  * v4 - pack all command structures, replace verbs wr/wc types with defined MIX types
+ * v5 - CM services with proxy_in, private data
  */
-#define DAT_MIX_VER            4
+#define DAT_MIX_VER            5
 #define DAT_MIX_MSG_MAX        256
 #define DAT_MIX_INLINE_MAX     256
 #define DAT_MIX_RDMA_MAX       (8*1024*1024)
@@ -449,6 +472,16 @@ typedef struct dat_mix_mr
 
 } __attribute__((packed)) dat_mix_mr_t;
 
+typedef struct dat_mix_mr_compat
+{
+       dat_mix_hdr_t           hdr;
+       uint32_t                mr_id;
+       uint32_t                len;
+       uint64_t                off;
+       uint64_t                ctx;
+
+} __attribute__((packed)) dat_mix_mr_compat_t;
+
 /***** MIX listen, status returned, no data *****/
 typedef struct dat_mix_listen
 {
@@ -533,6 +566,17 @@ typedef struct dat_mix_cm
 
 } dat_mix_cm_t;
 
+typedef struct dat_mix_cm_compat
+{
+       dat_mix_hdr_t           hdr;
+       uint64_t                sp_ctx;
+       uint64_t                cm_ctx;
+       uint32_t                cm_id;
+       uint32_t                qp_id;
+       dat_mcm_msg_compat_t    msg;
+
+} dat_mix_cm_compat_t;
+
 typedef struct dat_mix_cm_event
 {
        dat_mix_hdr_t           hdr;