mcm: add MFO support to openib_common code base

Provide full proxy support of CQ, QP, PZ, MR and device.
Use use new MXF_EP macro to switch proxy service based
on MXS (cross socket) or MFO (full offload) modes.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>
Signed-off-by: Amir Hanania <amir.hanania@intel.com>

diff --git a/dapl/openib_common/cq.c b/dapl/openib_common/cq.c
index a42d19dd49cb98935b5355e4190d94189048feb4..968cf446cf448334ff7a133b55dab2ea2a919a69 100644 (file)
--- a/dapl/openib_common/cq.c
+++ b/dapl/openib_common/cq.c
@@ -208,7 +208,7 @@ dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr,
                        goto err;
 
                /* cross-socket: shadow both RX and TX, no IB CQ on MIC */
-               if (MXS_EP(&ia_ptr->hca_ptr->ib_trans.addr))
+               if (MXF_EP(&ia_ptr->hca_ptr->ib_trans.addr))
                        return DAT_SUCCESS;
        }
        dapl_llist_init_entry(&evd_ptr->ib_cq_handle->entry);
@@ -584,7 +584,7 @@ DAT_RETURN dapls_ib_completion_poll(IN DAPL_HCA * hca_ptr,
                ep_ptr = cookie->ep;
                dapl_os_assert((NULL != ep_ptr));
                if ((!ep_ptr->qp_handle->tp->scif_ep) &&
-                   (ep_ptr->qp_handle->ep_map == MIC_XSOCK_DEV) &&
+                   (MXF_EP(ep_ptr->qp_handle)) &&
                    ((DAPL_GET_CQE_OPTYPE(wc_ptr) == OP_RDMA_WRITE_IMM) ||
                     (DAPL_GET_CQE_OPTYPE(wc_ptr) == OP_RDMA_WRITE))) {
                        dapl_log(DAPL_DBG_TYPE_EP,

diff --git a/dapl/openib_common/dapl_ib_dto.h b/dapl/openib_common/dapl_ib_dto.h
index de8fdc9de0458c583da9ffb54ffc4bf4480907f8..97341d2adb85d9bfd80a20f736b4641764575ed2 100644 (file)
--- a/dapl/openib_common/dapl_ib_dto.h
+++ b/dapl/openib_common/dapl_ib_dto.h
@@ -236,11 +236,10 @@ dapls_ib_post_send (
                 " post_snd: %s ep %p op %x flgs %x sgl %p,%d ln %d wr_id %Lx\n",
                 PROVIDER_NAME, ep_ptr, wr.opcode, wr.send_flags, wr.sg_list,
                 wr.num_sge, total_len, wr.wr_id);
-
 #ifdef _OPENIB_MCM_
        if (ep_ptr->qp_handle->tp->scif_ep)
                ret = dapli_mix_post_send(ep_ptr->qp_handle, total_len, &wr, &bad_wr);
-       else if (ep_ptr->qp_handle->ep_map == MIC_XSOCK_DEV)
+       else if (MXF_EP(ep_ptr->qp_handle))
                ret = mcm_send_pi(ep_ptr->qp_handle, total_len, &wr, &bad_wr);
        else
                ret = ibv_post_send(ep_ptr->qp_handle->qp2, &wr, &bad_wr);
@@ -448,7 +447,7 @@ dapls_ib_post_ext_send (
 #ifdef _OPENIB_MCM_
        if (ep_ptr->qp_handle->tp->scif_ep)
                ret = dapli_mix_post_send(ep_ptr->qp_handle, total_len, &wr, &bad_wr);
-       else if (ep_ptr->qp_handle->ep_map == MIC_XSOCK_DEV)
+       else if (MXF_EP(ep_ptr->qp_handle))
                ret = mcm_send_pi(ep_ptr->qp_handle, total_len, &wr, &bad_wr);
        else
                ret = ibv_post_send(ep_ptr->qp_handle->qp2, &wr, &bad_wr);

diff --git a/dapl/openib_common/mem.c b/dapl/openib_common/mem.c
index 0a82c000ae731fbcfbdd6396b07b1ac347b9d474..7f5ea6a4c55c762d78cae378c11c42c39edfd5cd 100644 (file)
--- a/dapl/openib_common/mem.c
+++ b/dapl/openib_common/mem.c
@@ -83,7 +83,13 @@ STATIC _INLINE_ int dapls_convert_privileges(IN DAT_MEM_PRIV_FLAGS privileges)
 DAT_RETURN dapls_ib_pd_alloc(IN DAPL_IA * ia_ptr, IN DAPL_PZ * pz)
 {
        /* get a protection domain */
-       pz->pd_handle = ibv_alloc_pd(ia_ptr->hca_ptr->ib_hca_handle);
+#ifdef _OPENIB_MCM_
+       if(MFO_EP(&ia_ptr->hca_ptr->ib_trans.addr) && ia_ptr->hca_ptr->ib_trans.scif_tx_ep)
+               dapli_mix_pz_create(ia_ptr, pz);
+       else
+#endif
+               pz->pd_handle = ibv_alloc_pd(ia_ptr->hca_ptr->ib_hca_handle);
+
        if (!pz->pd_handle)
                return (dapl_convert_errno(ENOMEM, "alloc_pd"));
 
@@ -112,11 +118,23 @@ DAT_RETURN dapls_ib_pd_alloc(IN DAPL_IA * ia_ptr, IN DAPL_PZ * pz)
  */
 DAT_RETURN dapls_ib_pd_free(IN DAPL_PZ * pz)
 {
+       int ret = DAT_INVALID_STATE;
+
        if (pz->pd_handle != IB_INVALID_HANDLE) {
-               ibv_dealloc_pd(pz->pd_handle);
+#ifdef _OPENIB_MCM_
+               DAPL_IA * ia_ptr = pz->header.owner_ia;
+               if (MFO_EP(&ia_ptr->hca_ptr->ib_trans.addr) && ia_ptr->hca_ptr->ib_trans.scif_tx_ep)
+                       ret = dapli_mix_pz_free(pz);
+               else
+#endif
+                       ret = ibv_dealloc_pd(pz->pd_handle);
+
                pz->pd_handle = IB_INVALID_HANDLE;
        }
-       return DAT_SUCCESS;
+       if (!ret)
+               return DAT_SUCCESS;
+
+       return DAT_INVALID_STATE;
 }
 
 /*
@@ -138,15 +156,15 @@ DAT_RETURN dapls_ib_pd_free(IN DAPL_PZ * pz)
  *     DAT_INSUFFICIENT_RESOURCES
  *
  */
-
 DAT_RETURN
 dapls_ib_mr_register(IN DAPL_IA * ia_ptr,
                     IN DAPL_LMR * lmr,
                     IN DAT_PVOID virt_addr,
                     IN DAT_VLEN length,
                     IN DAT_MEM_PRIV_FLAGS privileges, IN DAT_VA_TYPE va_type)
+#ifdef _OPENIB_MCM_
 {
-       struct ibv_device *ibv_dev = ia_ptr->hca_ptr->ib_hca_handle->device;
+       struct ibv_device *ibv_dev;
        int ib_access = dapls_convert_privileges(privileges);
 
        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
@@ -159,40 +177,37 @@ dapls_ib_mr_register(IN DAPL_IA * ia_ptr,
                return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
        }
 
-       /* iWARP only support */
-       if ((va_type == DAT_VA_TYPE_ZB) &&
-           (ibv_dev->transport_type != IBV_TRANSPORT_IWARP)) {
+       if (va_type == DAT_VA_TYPE_ZB) {
                dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                             " va_type == DAT_VA_TYPE_ZB: NOT SUPPORTED\n");
                return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
        }
 
-#ifdef _OPENIB_MCM_
        ib_access |= IBV_ACCESS_REMOTE_READ; /* HST->MXS, peer PI RR */
-#endif
 
        /* local read is default on IB */
-       lmr->mr_handle =
-           ibv_reg_mr(((DAPL_PZ *) lmr->param.pz_handle)->pd_handle,
-                      virt_addr, length, ib_access);
+       if (!MFO_EP(&ia_ptr->hca_ptr->ib_trans.addr) || !ia_ptr->hca_ptr->ib_trans.scif_tx_ep) {
+               lmr->mr_handle =
+                   ibv_reg_mr(((DAPL_PZ *) lmr->param.pz_handle)->pd_handle,
+                              virt_addr, length, ib_access);
 
-       if (!lmr->mr_handle)
-               return (dapl_convert_errno(ENOMEM, "reg_mr"));
+               if (!lmr->mr_handle)
+                       return (dapl_convert_errno(ENOMEM, "reg_mr"));
 
-       lmr->param.lmr_context = lmr->mr_handle->lkey;
-       lmr->param.rmr_context = lmr->mr_handle->rkey;
+               lmr->param.lmr_context = lmr->mr_handle->lkey;
+               lmr->param.rmr_context = lmr->mr_handle->rkey;
+
+               dapl_log(DAPL_DBG_TYPE_UTIL,
+                            " mr_register: mr=%p addr=%p pd %p ctx %p "
+                            "lkey=0x%x rkey=0x%x priv=%x\n",
+                            lmr->mr_handle, lmr->mr_handle->addr,
+                            lmr->mr_handle->pd, lmr->mr_handle->context,
+                            lmr->mr_handle->lkey, lmr->mr_handle->rkey,
+                            length, dapls_convert_privileges(privileges));
+       }
        lmr->param.registered_size = length;
        lmr->param.registered_address = (DAT_VADDR) (uintptr_t) virt_addr;
 
-       dapl_log(DAPL_DBG_TYPE_UTIL,
-                    " mr_register: mr=%p addr=%p pd %p ctx %p "
-                    "lkey=0x%x rkey=0x%x priv=%x\n",
-                    lmr->mr_handle, lmr->mr_handle->addr,
-                    lmr->mr_handle->pd, lmr->mr_handle->context,
-                    lmr->mr_handle->lkey, lmr->mr_handle->rkey,
-                    length, dapls_convert_privileges(privileges));
-
-#ifdef _OPENIB_MCM_
        /* RDMA proxy with MPXYD register with SCIF */
        if (ia_ptr->hca_ptr->ib_trans.scif_tx_ep) {
                DAT_VLEN offset = (DAT_VLEN)(virt_addr - ALIGN_DOWN_PPAGE(virt_addr));
@@ -218,20 +233,76 @@ dapls_ib_mr_register(IN DAPL_IA * ia_ptr,
                             mcm_map_str(ia_ptr->hca_ptr->ib_trans.addr.ep_map));
 
                /* plus cross socket proxy-in; send registration info to MPXYD */
-               if (MXS_EP(&ia_ptr->hca_ptr->ib_trans.addr)) {
+               if (MXF_EP(&ia_ptr->hca_ptr->ib_trans.addr)) {
+                       /* MFO - Mic in Mic Full Offload */
                        if (dapli_mix_mr_create(&ia_ptr->hca_ptr->ib_trans, lmr)) {
                                dapls_ib_mr_deregister(lmr);
                                return (dapl_convert_errno(ENOMEM, "reg_mr"));
                        }
+                       if (MFO_EP(&ia_ptr->hca_ptr->ib_trans.addr)) {
+                               lmr->param.lmr_context = lmr->mr_id;
+                               lmr->param.rmr_context = lmr->mr_id;
+                               lmr->mr_handle->lkey = lmr->param.lmr_context;
+                               lmr->mr_handle->rkey = lmr->param.rmr_context;
+                       }
                }
        }
-#endif
        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                     " mr_register: ia=%p, lmr=%p va=%p ln=%d priv=%d return\n",
                     ia_ptr, lmr, virt_addr, length, privileges);
 
        return DAT_SUCCESS;
 }
+#else
+{
+       struct ibv_device *ibv_dev;
+       int ib_access = dapls_convert_privileges(privileges);
+
+       dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+                    " mr_register: ia=%p, lmr=%p va=%p ln=%d pv=0x%x\n",
+                    ia_ptr, lmr, virt_addr, length, privileges);
+
+       if (lmr->param.mem_type == DAT_MEM_TYPE_SHARED_VIRTUAL) {
+               dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+                            " mr_register_shared: NOT IMPLEMENTED\n");
+               return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+       }
+
+       /* iWARP only support */
+       ibv_dev = ia_ptr->hca_ptr->ib_hca_handle->device;
+       if ((va_type == DAT_VA_TYPE_ZB) && (ibv_dev->transport_type != IBV_TRANSPORT_IWARP)) {
+               dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+                    " va_type == DAT_VA_TYPE_ZB: NOT SUPPORTED\n");
+               return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+       }
+
+       /* local read is default on IB */
+       lmr->mr_handle = ibv_reg_mr(((DAPL_PZ *)lmr->param.pz_handle)->pd_handle,
+                                   virt_addr, length, ib_access);
+       if (!lmr->mr_handle)
+               return (dapl_convert_errno(ENOMEM, "reg_mr"));
+
+       lmr->param.lmr_context = lmr->mr_handle->lkey;
+       lmr->param.rmr_context = lmr->mr_handle->rkey;
+
+       dapl_log(DAPL_DBG_TYPE_UTIL,
+                    " mr_register: mr=%p addr=%p pd %p ctx %p "
+                    "lkey=0x%x rkey=0x%x priv=%x\n",
+                    lmr->mr_handle, lmr->mr_handle->addr,
+                    lmr->mr_handle->pd, lmr->mr_handle->context,
+                    lmr->mr_handle->lkey, lmr->mr_handle->rkey,
+                    length, dapls_convert_privileges(privileges));
+
+       lmr->param.registered_size = length;
+       lmr->param.registered_address = (DAT_VADDR) (uintptr_t) virt_addr;
+
+       dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+                    " mr_register: ia=%p, lmr=%p va=%p ln=%d priv=%d return\n",
+                    ia_ptr, lmr, virt_addr, length, privileges);
+
+       return DAT_SUCCESS;
+}
+#endif
 
 /*
  * dapl_ib_mr_deregister
@@ -250,20 +321,22 @@ dapls_ib_mr_register(IN DAPL_IA * ia_ptr,
  *
  */
 DAT_RETURN dapls_ib_mr_deregister(IN DAPL_LMR * lmr)
-{
 #ifdef _OPENIB_MCM_
-       if (lmr->header.owner_ia->hca_ptr->ib_trans.scif_ep) {
+{
+       DAPL_IA * ia_ptr = (DAPL_IA *)lmr->param.ia_handle;
+       if (ia_ptr->hca_ptr->ib_trans.scif_ep) {
                int ret;
 
-               if (lmr->header.owner_ia->hca_ptr->ib_trans.addr.ep_map == MIC_XSOCK_DEV) {
-                       if (dapli_mix_mr_free(&lmr->header.owner_ia->hca_ptr->ib_trans, lmr)) {
+               if (MXF_EP(&ia_ptr->hca_ptr->ib_trans.addr)) {
+                       if (dapli_mix_mr_free(&ia_ptr->hca_ptr->ib_trans, lmr)) {
                                dapl_log(DAPL_DBG_TYPE_ERR,
                                         " mr_deregister: ERR: lmr_remove; id %d sci_addr %p, off 0x%llx, len %d ret=%s\n",
                                         lmr->mr_id, lmr->param.registered_address, lmr->sci_addr,
                                         ALIGN_PAGE(lmr->param.registered_size+lmr->sci_off), strerror(errno));
+                               return DAT_INVALID_STATE;
                        }
                }
-               ret = scif_unregister(lmr->header.owner_ia->hca_ptr->ib_trans.scif_tx_ep,
+               ret = scif_unregister(ia_ptr->hca_ptr->ib_trans.scif_tx_ep,
                                      lmr->sci_addr, ALIGN_PAGE(lmr->param.registered_size+lmr->sci_off));
                if (ret)
                        dapl_log(DAPL_DBG_TYPE_ERR,
@@ -276,15 +349,24 @@ DAT_RETURN dapls_ib_mr_deregister(IN DAPL_LMR * lmr)
                lmr->sci_off = 0;
                lmr->mr_id = 0;
        }
-#endif
        if (lmr->mr_handle != IB_INVALID_HANDLE) {
                if (ibv_dereg_mr(lmr->mr_handle))
                        return (dapl_convert_errno(errno, "dereg_mr"));
                lmr->mr_handle = IB_INVALID_HANDLE;
        }
+       return DAT_SUCCESS;
+}
+#else
+{
+       if (lmr->mr_handle != IB_INVALID_HANDLE) {
+               if (ibv_dereg_mr(lmr->mr_handle))
+                       return (dapl_convert_errno(errno, "dereg_mr"));
 
+               lmr->mr_handle = IB_INVALID_HANDLE;
+       }
        return DAT_SUCCESS;
 }
+#endif
 
 /*
  * dapl_ib_mr_register_shared

diff --git a/dapl/openib_common/qp.c b/dapl/openib_common/qp.c
index d3584e5c1bd39baa8868083dae35f520126d59c5..527fc1d4c46b62b4ca3f95a8471619a755056e7e 100644 (file)
--- a/dapl/openib_common/qp.c
+++ b/dapl/openib_common/qp.c
@@ -223,7 +223,7 @@ dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,
        /* Don't create any QP if MIC xsocket, QPt and QPr both on MPXYD */
        if (!ia_ptr->hca_ptr->ib_trans.scif_ep ||
            (ia_ptr->hca_ptr->ib_trans.scif_ep &&
-           !MXS_EP(&ia_ptr->hca_ptr->ib_trans.addr))) {
+           !MXF_EP(&ia_ptr->hca_ptr->ib_trans.addr))) {
                /* QP1 needed for RX only, set QP1 req_cq empty */
                qp_create.send_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty->cq;
        } else {

diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c
index c44f78ae903d69d5c09fb5ea32c488c4a2ae5bb1..55bda3b6c8d3fe72eadfaef8debb4dd91a94ab90 100644 (file)
--- a/dapl/openib_common/util.c
+++ b/dapl/openib_common/util.c
@@ -407,12 +407,19 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
                tp->na.mode = "DIRECT";
                tp->na.read = "TRUE";
 #endif
-
-       /* query verbs for this device and port attributes */
-       if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) ||
-           ibv_query_port(hca_ptr->ib_hca_handle,
-                          hca_ptr->port_num, &port_attr))
-               return (dapl_convert_errno(errno, "ib_query_hca"));
+#ifdef _OPENIB_MCM_
+       if (MFO_EP(&hca_ptr->ib_trans.addr) && tp->self.node) {
+               if (dapli_mix_query_device(tp, &dev_attr) ||
+                       dapli_mix_query_port(tp, hca_ptr->port_num, &port_attr))
+                               return (dapl_convert_errno(errno, "mix_query_hca"));
+       }
+       else
+#endif
+               /* query verbs for this device and port attributes */
+               if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) ||
+                               ibv_query_port(hca_ptr->ib_hca_handle,
+                                               hca_ptr->port_num, &port_attr))
+                       return (dapl_convert_errno(errno, "ib_query_hca"));
 
        dev_attr.max_qp_wr = DAPL_MIN(dev_attr.max_qp_wr,
                                      dapl_os_get_env_val("DAPL_WR_MAX", dev_attr.max_qp_wr));
@@ -428,9 +435,16 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
 
        if (ia_attr != NULL) {
                (void)dapl_os_memzero(ia_attr, sizeof(*ia_attr));
-               strncpy(ia_attr->adapter_name,
-                       ibv_get_device_name(tp->ib_dev),
-                       DAT_NAME_MAX_LENGTH - 1);
+#ifdef _OPENIB_MCM_
+               /* MIC at MFO mode - Initialized at Open */
+               if (MFO_EP(&hca_ptr->ib_trans.addr) && tp->self.node) {
+                       strncpy(ia_attr->adapter_name, hca_ptr->name, DAT_NAME_MAX_LENGTH);
+               }
+               else
+#endif
+                       strncpy(ia_attr->adapter_name, ibv_get_device_name(tp->ib_dev),
+                                       DAT_NAME_MAX_LENGTH - 1);
+
                ia_attr->adapter_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
                ia_attr->vendor_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
                ia_attr->ia_address_ptr = (DAT_IA_ADDRESS_PTR) &hca_ptr->hca_address;
@@ -465,10 +479,12 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
                ia_attr->max_message_size = port_attr.max_msg_sz;
                ia_attr->max_rdma_size = port_attr.max_msg_sz;
                /* iWARP spec. - 1 sge for RDMA reads */
-               if (hca_ptr->ib_hca_handle->device->transport_type == IBV_TRANSPORT_IWARP)
-                       ia_attr->max_iov_segments_per_rdma_read = 1;
-               else
-                       ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge;
+               ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge;
+#ifdef _OPENIB_MCM_
+               if (!MFO_EP(&hca_ptr->ib_trans.addr))
+#endif
+                       if (hca_ptr->ib_hca_handle->device->transport_type == IBV_TRANSPORT_IWARP)
+                               ia_attr->max_iov_segments_per_rdma_read = 1;
                ia_attr->max_iov_segments_per_rdma_write = dev_attr.max_sge;
                ia_attr->num_transport_attr = 0;
                ia_attr->transport_attr = NULL;
@@ -489,8 +505,14 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
                tp->na.mtu = dapl_ib_mtu_str(tp->ib_cm.mtu);
                tp->na.port = dapl_ib_port_str(port_attr.state);
                tp->na.port_num = dapl_ib_port_num_str(hca_ptr->port_num);
-               if (!tp->guid)
-                       tp->guid = ntohll(ibv_get_device_guid(tp->ib_dev));
+               if (!tp->guid) {
+#ifdef _OPENIB_MCM_
+                       if (MFO_EP(&hca_ptr->ib_trans.addr) && tp->self.node)
+                               tp->guid = ntohll(dev_attr.node_guid);
+                       else
+#endif
+                               tp->guid = ntohll(ibv_get_device_guid(tp->ib_dev));
+               }
 
                sprintf(tp->guid_str, "%04x:%04x:%04x:%04x",
                        (unsigned) (tp->guid >> 48) & 0xffff,
@@ -498,6 +520,12 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
                        (unsigned) (tp->guid >> 16) & 0xffff,
                        (unsigned) (tp->guid >>  0) & 0xffff);
 
+#ifdef _OPENIB_MCM_
+               /* initialized at open */
+               if (MFO_EP(&hca_ptr->ib_trans.addr) && tp->self.node)
+                       goto skip_ib;
+#endif
+
                if (hca_ptr->ib_hca_handle->device->transport_type != IBV_TRANSPORT_IB)
                        goto skip_ib;