dapl scm: add mtu adjustments via environment, default = 1024.

DAPL_IB_MTU adjusts path mtu setting for RC qp's. Default setting
is min of 1024 and active mtu on IB device.

Signed-off by: Arlin Davis ardavis@ichips.intel.com

diff --git a/dapl/openib_scm/dapl_ib_util.c b/dapl/openib_scm/dapl_ib_util.c
index 0f24737d26b3197c584a09953e0af953323b9567..f7ee58b760a9d9ff056f95418481a2cba85b162e 100644 (file)
--- a/dapl/openib_scm/dapl_ib_util.c
+++ b/dapl/openib_scm/dapl_ib_util.c
@@ -64,6 +64,18 @@ static const char rcsid[] = "$Id:  $";
 int g_dapl_loopback_connection = 0;
 int g_scm_pipe[2];
 
+enum ibv_mtu dapl_ib_mtu(int mtu)
+{
+       switch (mtu) {
+       case 256:  return IBV_MTU_256;
+       case 512:  return IBV_MTU_512;
+       case 1024: return IBV_MTU_1024;
+       case 2048: return IBV_MTU_2048;
+       case 4096: return IBV_MTU_4096;
+       default:   return IBV_MTU_1024;
+       }
+}
+
 /* just get IP address for hostname */
 DAT_RETURN getipaddr( char *addr, int addr_len)
 {
@@ -223,6 +235,8 @@ found:
                dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT);
        hca_ptr->ib_trans.tclass =
                dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS);
+       hca_ptr->ib_trans.mtu =
+               dapl_ib_mtu(dapl_os_get_env_val("DAPL_IB_MTU", SCM_IB_MTU));
 
        /* initialize cq_lock */
        dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock);
@@ -448,19 +462,24 @@ DAT_RETURN dapls_ib_query_hca (
                 ia_attr->extension_supported = DAT_EXTENSION_IB;
                 ia_attr->extension_version = DAT_IB_EXTENSION_VERSION;
 #endif
-               hca_ptr->ib_trans.mtu = port_attr.active_mtu;
+               hca_ptr->ib_trans.mtu = DAPL_MIN(port_attr.active_mtu,
+                                                hca_ptr->ib_trans.mtu);
                hca_ptr->ib_trans.ack_timer = 
                        DAPL_MAX(dev_attr.local_ca_ack_delay,
                                 hca_ptr->ib_trans.ack_timer);
 
                dapl_dbg_log (DAPL_DBG_TYPE_UTIL, 
-                       " query_hca: (%x.%x) ep %d ep_q %d evd %d evd_q %d\n", 
+                       " query_hca: (%x.%x) ep %d ep_q %d evd %d"
+                       " evd_q %d mtu %d\n", 
                        ia_attr->hardware_version_major,
                        ia_attr->hardware_version_minor,
                        ia_attr->max_eps, ia_attr->max_dto_per_ep,
-                       ia_attr->max_evds, ia_attr->max_evd_qlen );
+                       ia_attr->max_evds, ia_attr->max_evd_qlen,
+                       128 << hca_ptr->ib_trans.mtu);
+
                dapl_dbg_log (DAPL_DBG_TYPE_UTIL, 
-                       " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d ack_time %d\n", 
+                       " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d"
+                       " ack_time %d\n", 
                        ia_attr->max_message_size, ia_attr->max_rdma_size,
                        ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs, 
                        ia_attr->max_rmrs,hca_ptr->ib_trans.ack_timer );

diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h
index deb6be331b770f5f68b62770894e203a9eddf4c9..bd702a92fb3a0ac2da2a7ce843471054b78a1559 100644 (file)
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -196,6 +196,7 @@ typedef struct ibv_comp_channel *ib_wait_obj_handle_t;
 #define SCM_ACK_RETRY 7                /* 3 bits, 7 * 134ms = 940ms */
 #define SCM_RNR_TIMER 28       /* 5 bits, 28 == 163ms, 31 == 491ms */
 #define SCM_RNR_RETRY 7                /* 3 bits, 7 == infinite */
+#define SCM_IB_MTU    1024
 
 /* Global routing defaults */
 #define SCM_GLOBAL     0       /* global routing is disabled */