From 70e47f49465fb182133efa0d05e7b13e568da22c Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Thu, 12 Sep 2013 09:12:55 -0700 Subject: [PATCH] mcm: reduce max qp depth and msg size in proxy mode, allow override DAPL_MCM_WR_MAX is used set max qp depth on mcm provider, default=500 DAPL_MCM_MSG_MAX is used set max msg size on mcm provider, default=8388608 DAPL_WR_MAX is used to override max qp depth on all IB providers. Signed-off-by: Arlin Davis --- dapl/openib_common/util.c | 16 ++++++++++------ dat/include/dat2/dat_mic_extensions.h | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c index 626e7b8..bfd4081 100644 --- a/dapl/openib_common/util.c +++ b/dapl/openib_common/util.c @@ -312,13 +312,19 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, hca_ptr->port_num, &port_attr)) return (dapl_convert_errno(errno, "ib_query_hca")); + dev_attr.max_qp_wr = DAPL_MIN(dev_attr.max_qp_wr, + dapl_os_get_env_val("DAPL_WR_MAX", dev_attr.max_qp_wr)); + #ifdef _OPENIB_MCM_ - /* Adjust for CCL Proxy; limited sge and no READ support */ + /* Adjust for CCL Proxy; limited sge's, no READ support, reduce QP and RDMA limits */ if (hca_ptr->ib_trans.scif_ep) { dev_attr.max_sge = DAPL_MIN(dev_attr.max_sge, DAT_MIX_SGE_MAX); + dev_attr.max_qp_wr = DAPL_MIN(dev_attr.max_qp_wr, + dapl_os_get_env_val("DAPL_MCM_WR_MAX", DAT_MIX_WR_MAX)); + port_attr.max_msg_sz = DAPL_MIN(port_attr.max_msg_sz, + dapl_os_get_env_val("DAPL_MCM_MSG_MAX", DAT_MIX_RDMA_MAX)); } #endif - if (ia_attr != NULL) { (void)dapl_os_memzero(ia_attr, sizeof(*ia_attr)); strncpy(ia_attr->adapter_name, @@ -359,12 +365,10 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, ia_attr->max_message_size = port_attr.max_msg_sz; ia_attr->max_rdma_size = port_attr.max_msg_sz; /* iWARP spec. - 1 sge for RDMA reads */ - if (hca_ptr->ib_hca_handle->device->transport_type - == IBV_TRANSPORT_IWARP) + if (hca_ptr->ib_hca_handle->device->transport_type == IBV_TRANSPORT_IWARP) ia_attr->max_iov_segments_per_rdma_read = 1; else - ia_attr->max_iov_segments_per_rdma_read = - dev_attr.max_sge; + ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge; ia_attr->max_iov_segments_per_rdma_write = dev_attr.max_sge; ia_attr->num_transport_attr = 0; ia_attr->transport_attr = NULL; diff --git a/dat/include/dat2/dat_mic_extensions.h b/dat/include/dat2/dat_mic_extensions.h index eca9bc9..975e221 100755 --- a/dat/include/dat2/dat_mic_extensions.h +++ b/dat/include/dat2/dat_mic_extensions.h @@ -202,6 +202,8 @@ typedef struct dat_mcm_msg #define DAT_MIX_VER 4 #define DAT_MIX_MSG_MAX 256 #define DAT_MIX_INLINE_MAX 256 +#define DAT_MIX_RDMA_MAX (8*1024*1024) +#define DAT_MIX_WR_MAX 500 typedef enum dat_mix_ops { -- 2.46.0