dapl scm: support global routing and set mtu based on active_mtu

Signed-off by: Arlin Davis ardavis@ichips.intel.com

diff --git a/dapl/openib_scm/dapl_ib_qp.c b/dapl/openib_scm/dapl_ib_qp.c
index 1eba2bd334b7536566e05d399ff2a6830ee145a5..8577131821e6646834b28d676f36095bd2582aaa 100644 (file)
--- a/dapl/openib_scm/dapl_ib_qp.c
+++ b/dapl/openib_scm/dapl_ib_qp.c
@@ -25,9 +25,9 @@
 
 /**********************************************************************
  *
- * MODULE: dapl_det_qp.c
+ * MODULE: dapl_ib_qp.c
  *
- * PURPOSE: QP routines for access to DET Verbs
+ * PURPOSE: QP routines for access to ofa rdma verbs 
  *
  * $Id: $
  **********************************************************************/
@@ -285,17 +285,12 @@ void
 dapls_ib_reinit_ep (
        IN  DAPL_EP     *ep_ptr)
 {
-       
        if ( ep_ptr->qp_handle != IB_INVALID_HANDLE ) {
                /* move to RESET state and then to INIT */
                dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_RESET, 0);
                dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_INIT, 0);
                ep_ptr->qp_state = IB_QP_STATE_INIT;
        }
-
-       /* TODO: When IB-CM is implement then handle timewait before 
-        * allowing re-use of this QP
-        */
 }
 
 /* 
@@ -325,19 +320,28 @@ dapls_modify_qp_state ( IN ib_qp_handle_t qp_handle,
                                IBV_QP_MAX_DEST_RD_ATOMIC |
                                IBV_QP_MIN_RNR_TIMER;
 
-                       qp_attr.qp_state                = IBV_QPS_RTR;
-                       qp_attr.path_mtu                = IBV_MTU_2048;
-                       qp_attr.dest_qp_num             = qp_cm->qpn;
-                       qp_attr.rq_psn                  = 1;
-                       qp_attr.max_dest_rd_atomic      = 
+                       qp_attr.qp_state = IBV_QPS_RTR;
+                       qp_attr.dest_qp_num = qp_cm->qpn;
+                       qp_attr.rq_psn = 1;
+                       qp_attr.path_mtu = 
+                               ia_ptr->hca_ptr->ib_trans.mtu;
+                       qp_attr.max_dest_rd_atomic = 
                                ep_ptr->param.ep_attr.max_rdma_read_out;
-                       qp_attr.min_rnr_timer           = ia_ptr->hca_ptr->ib_trans.rnr_timer;
-                       qp_attr.ah_attr.is_global       = 0;
-                       qp_attr.ah_attr.grh.dgid        = qp_cm->gid;
-                       qp_attr.ah_attr.dlid            = qp_cm->lid;
-                       qp_attr.ah_attr.sl              = 0;
-                       qp_attr.ah_attr.src_path_bits   = 0;
-                       qp_attr.ah_attr.port_num        = qp_cm->port;
+                       qp_attr.min_rnr_timer =
+                               ia_ptr->hca_ptr->ib_trans.rnr_timer;
+                       qp_attr.ah_attr.dlid = qp_cm->lid;
+                       /* global routing */
+                       if (ia_ptr->hca_ptr->ib_trans.global) {
+                               qp_attr.ah_attr.is_global = 1;
+                               qp_attr.ah_attr.grh.dgid = qp_cm->gid;
+                               qp_attr.ah_attr.grh.hop_limit = 
+                                               ia_ptr->hca_ptr->ib_trans.hop_limit;
+                               qp_attr.ah_attr.grh.traffic_class = 
+                                               ia_ptr->hca_ptr->ib_trans.tclass;
+                       }
+                       qp_attr.ah_attr.sl = 0;
+                       qp_attr.ah_attr.src_path_bits = 0;
+                       qp_attr.ah_attr.port_num = qp_cm->port;
                        
                        dapl_dbg_log (DAPL_DBG_TYPE_EP,
                              " modify_qp_rtr: qpn %x lid %x "

diff --git a/dapl/openib_scm/dapl_ib_util.c b/dapl/openib_scm/dapl_ib_util.c
index 9bf0e2bac12ba74a122fecc8f178072e1a0467b0..9ce9e6a9a6f772591d7e610ff11b406707319de1 100644 (file)
--- a/dapl/openib_scm/dapl_ib_util.c
+++ b/dapl/openib_scm/dapl_ib_util.c
@@ -210,6 +210,12 @@ found:
                dapl_os_get_env_val("DAPL_RNR_RETRY", SCM_RNR_RETRY);
        hca_ptr->ib_trans.rnr_timer = 
                dapl_os_get_env_val("DAPL_RNR_TIMER", SCM_RNR_TIMER);
+       hca_ptr->ib_trans.global =
+               dapl_os_get_env_val("DAPL_GLOBAL_ROUTING", SCM_GLOBAL);
+       hca_ptr->ib_trans.hop_limit =
+               dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT);
+       hca_ptr->ib_trans.tclass =
+               dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS);
 
        /* initialize cq_lock */
        dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock);
@@ -437,6 +443,7 @@ DAT_RETURN dapls_ib_query_hca (
                 ia_attr->extension_supported = DAT_EXTENSION_IB;
                 ia_attr->extension_version = DAT_IB_EXTENSION_VERSION;
 #endif
+               hca_ptr->ib_trans.mtu = port_attr.active_mtu;
                hca_ptr->ib_trans.ack_timer = 
                        DAPL_MAX(dev_attr.local_ca_ack_delay,
                                 hca_ptr->ib_trans.ack_timer);

diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h
index fd9cd2ffadd0c1a07f951e7f2746bdc7f2c831a3..39eb2457f228f9fcc9cd70a8dd3ce7344ecaedd7 100644 (file)
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -186,6 +186,11 @@ typedef struct ibv_comp_channel *ib_wait_obj_handle_t;
 #define SCM_RNR_TIMER 28       /* 5 bits, 28 == 163ms, 31 == 491ms */
 #define SCM_RNR_RETRY 7                /* 3 bits, 7 == infinite */
 
+/* Global routing defaults */
+#define SCM_GLOBAL     0       /* global routing is disabled */
+#define SCM_HOP_LIMIT  0xff
+#define SCM_TCLASS     0
+
 /* CM private data areas */
 #define        IB_MAX_REQ_PDATA_SIZE   92
 #define        IB_MAX_REP_PDATA_SIZE   196
@@ -193,7 +198,7 @@ typedef struct ibv_comp_channel *ib_wait_obj_handle_t;
 #define        IB_MAX_DREQ_PDATA_SIZE  220
 #define        IB_MAX_DREP_PDATA_SIZE  224
 
-/* DTO OPs, ordered for DAPL ENUM definitions ???*/
+/* DTO OPs, ordered for DAPL ENUM definitions */
 #define OP_RDMA_WRITE           IBV_WR_RDMA_WRITE
 #define OP_RDMA_WRITE_IMM       IBV_WR_RDMA_WRITE_WITH_IMM
 #define OP_SEND                 IBV_WR_SEND
@@ -295,6 +300,10 @@ typedef struct _ib_hca_transport
        uint8_t                 ack_retry;
        uint8_t                 rnr_timer;
        uint8_t                 rnr_retry;
+       uint8_t                 global;
+       uint8_t                 hop_limit;
+       uint8_t                 tclass;
+       uint8_t                 mtu;
 } ib_hca_transport_t;
 
 /* provider specfic fields for shared memory support */