commom: add lmr support for proxy in service

Registration details must be tranfered to proxy service
to enable proxy-in data transfers. IB registration
and SCIF registration is sent to mpxyd for inbound
rdma write TPT services for IB RW store and SCIF writeto
forward capabilities. Extend DAT LMR to include
scif information and ID. If proxy service is
in use call new functions dapli_mix_mr_create/free
to sync with mpxyd.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/include/dapl.h b/dapl/include/dapl.h
index 26bf5d8a0f95ca821569cb943132d5fe21cd7655..09b7a987761f2f130a017048fc1aee3ea4b1476c 100755 (executable)
--- a/dapl/include/dapl.h
+++ b/dapl/include/dapl.h
@@ -495,12 +495,13 @@ struct dapl_lmr
 {
     DAPL_HEADER                header;
     DAT_LMR_PARAM      param;
-#ifdef _OPENIB_MCM_
-    off_t              scif_off;
-    int                        offset;
-#endif
     ib_mr_handle_t     mr_handle;
     DAPL_ATOMIC                lmr_ref_count;
+#ifdef _OPENIB_MCM_
+    off_t              sci_addr;
+    int                        sci_off;
+    int                        mr_id;
+#endif
 #if !defined(__KDAPL__)
     char               shmid[DAT_LMR_COOKIE_SIZE]; /* shared memory ID */
     ib_shm_transport_t ib_trans;       /* provider specific data */

diff --git a/dapl/openib_common/mem.c b/dapl/openib_common/mem.c
index ee4d569f522d4d8da907c95f69b01cc93c83ea48..a0ebf6fa7d62845ccaa9e7ceee4146645e3b721c 100644 (file)
--- a/dapl/openib_common/mem.c
+++ b/dapl/openib_common/mem.c
@@ -167,36 +167,6 @@ dapls_ib_mr_register(IN DAPL_IA * ia_ptr,
                return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
        }
 
-#ifdef _OPENIB_MCM_
-       /* RDMA proxy with MPXYD? register with SCIF
-        *
-        * TODO: is there some way to determine out bound writes only with READ only permission?
-        * I doubt if consumers actually set permissions read only for RDMA writes out.
-        * For now we have to register everything.
-        */
-       if (ia_ptr->hca_ptr->ib_trans.scif_tx_ep) {
-               DAT_VLEN offset = virt_addr - ALIGN_DOWN_PPAGE(virt_addr);
-               lmr->scif_off = scif_register(ia_ptr->hca_ptr->ib_trans.scif_tx_ep,
-                                             (void*)ALIGN_DOWN_PPAGE(virt_addr),
-                                             ALIGN_PAGE(length + offset),
-                                             (off_t)0,
-                                             SCIF_PROT_READ | SCIF_PROT_WRITE,
-                                             0);
-               if (lmr->scif_off == (off_t)(-1)) {
-                       dapl_log(DAPL_DBG_TYPE_ERR,
-                                " mr_register: SCIF_mr v_addr=%p, %p len=%d ep=%d, failed %s\n",
-                                virt_addr, (void*)ALIGN_DOWN_PPAGE(virt_addr),
-                                ALIGN_PAGE(length + offset),
-                                ia_ptr->hca_ptr->ib_trans.scif_ep, strerror(errno));
-                       return (dapl_convert_errno(ENOMEM, "reg_mr_scif"));
-               }
-               lmr->offset = offset;
-               dapl_log(DAPL_DBG_TYPE_UTIL,
-                            " mr_register: SCIF_mr addr %p, scif_off 0x%llx, offset 0x%x len %d\n",
-                            (void*)ALIGN_DOWN_PPAGE(virt_addr), lmr->scif_off,
-                            lmr->offset, ALIGN_PAGE(length + offset));
-       }
-#endif
        /* local read is default on IB */
        lmr->mr_handle =
            ibv_reg_mr(((DAPL_PZ *) lmr->param.pz_handle)->pd_handle,
@@ -218,8 +188,40 @@ dapls_ib_mr_register(IN DAPL_IA * ia_ptr,
                     lmr->mr_handle->lkey, lmr->mr_handle->rkey,
                     length, dapls_convert_privileges(privileges));
 
-
-
+#ifdef _OPENIB_MCM_
+       /* RDMA proxy with MPXYD register with SCIF */
+       if (ia_ptr->hca_ptr->ib_trans.scif_tx_ep) {
+               DAT_VLEN offset = (DAT_VLEN)(virt_addr - ALIGN_DOWN_PPAGE(virt_addr));
+               lmr->sci_addr = scif_register(ia_ptr->hca_ptr->ib_trans.scif_tx_ep,
+                                             (void*)ALIGN_DOWN_PPAGE(virt_addr),
+                                             ALIGN_PAGE(length + offset),
+                                             (off_t)0,
+                                             SCIF_PROT_READ | SCIF_PROT_WRITE,
+                                             0);
+               if (lmr->sci_addr == (off_t)(-1)) {
+                       dapl_log(DAPL_DBG_TYPE_ERR,
+                                " mr_reg: SCIF va=%p, %p len=%d ep=%d, %s\n",
+                                virt_addr, (void*)ALIGN_DOWN_PPAGE(virt_addr),
+                                ALIGN_PAGE(length + offset),
+                                ia_ptr->hca_ptr->ib_trans.scif_tx_ep, strerror(errno));
+                       return (dapl_convert_errno(ENOMEM, "reg_mr_scif"));
+               }
+               lmr->sci_off = offset;
+               dapl_log(DAPL_DBG_TYPE_UTIL,
+                            " mr_reg: SCIF va %p, scif_addr 0x%llx, off 0x%x ln %d %s\n",
+                            (void*)ALIGN_DOWN_PPAGE(virt_addr), lmr->sci_addr,
+                            lmr->sci_off, ALIGN_PAGE(length + offset),
+                            mcm_map_str(ia_ptr->hca_ptr->ib_trans.addr.ep_map));
+
+               /* plus cross socket proxy-in; send registration info to MPXYD */
+               if (MXS_EP(&ia_ptr->hca_ptr->ib_trans.addr)) {
+                       if (dapli_mix_mr_create(&ia_ptr->hca_ptr->ib_trans, lmr)) {
+                               dapls_ib_mr_deregister(lmr);
+                               return (dapl_convert_errno(ENOMEM, "reg_mr"));
+                       }
+               }
+       }
+#endif
        return DAT_SUCCESS;
 }
 
@@ -241,25 +243,38 @@ dapls_ib_mr_register(IN DAPL_IA * ia_ptr,
  */
 DAT_RETURN dapls_ib_mr_deregister(IN DAPL_LMR * lmr)
 {
+#ifdef _OPENIB_MCM_
+       if (lmr->header.owner_ia->hca_ptr->ib_trans.scif_ep) {
+               int ret;
+
+               if (lmr->header.owner_ia->hca_ptr->ib_trans.addr.ep_map == MIC_XSOCK_DEV) {
+                       if (dapli_mix_mr_free(&lmr->header.owner_ia->hca_ptr->ib_trans, lmr)) {
+                               dapl_log(DAPL_DBG_TYPE_ERR,
+                                        " mr_deregister: ERR: lmr_remove; id %d sci_addr %p, off 0x%llx, len %d ret=%s\n",
+                                        lmr->mr_id, lmr->param.registered_address, lmr->sci_addr,
+                                        ALIGN_PAGE(lmr->param.registered_size+lmr->sci_off), strerror(errno));
+                       }
+               }
+               ret = scif_unregister(lmr->header.owner_ia->hca_ptr->ib_trans.scif_tx_ep,
+                                     lmr->sci_addr, ALIGN_PAGE(lmr->param.registered_size+lmr->sci_off));
+               if (ret)
+                       dapl_log(DAPL_DBG_TYPE_ERR,
+                            " mr_deregister: ERR: SCIF_unreg addr %p, off 0x%llx, len %d ret=%s\n",
+                            lmr->param.registered_address, lmr->sci_off,
+                            ALIGN_PAGE(lmr->param.registered_size+lmr->sci_off),
+                            strerror(errno));
+
+               lmr->sci_addr = 0;
+               lmr->sci_off = 0;
+               lmr->mr_id = 0;
+       }
+#endif
        if (lmr->mr_handle != IB_INVALID_HANDLE) {
                if (ibv_dereg_mr(lmr->mr_handle))
                        return (dapl_convert_errno(errno, "dereg_mr"));
                lmr->mr_handle = IB_INVALID_HANDLE;
        }
 
-#ifdef _OPENIB_MCM_
-       if (lmr->header.owner_ia->hca_ptr->ib_trans.scif_ep) {
-               if(scif_unregister(lmr->header.owner_ia->hca_ptr->ib_trans.scif_tx_ep,
-                                  lmr->scif_off, ALIGN_PAGE(lmr->param.registered_size+lmr->offset)))
-                       return (dapl_convert_errno(errno, "dereg_mr_scif"));
-
-               dapl_log(DAPL_DBG_TYPE_UTIL,
-                            " mr_deregister: SCIF addr %p, off 0x%llx, len %d\n",
-                            lmr->param.registered_address, lmr->scif_off,
-                            ALIGN_PAGE(lmr->param.registered_size+lmr->offset));
-       }
-#endif
-
        return DAT_SUCCESS;
 }