openib: add inline data support check during device open

Not all rdma devices support inline data, however without
a verbs device attribute the only way to determine
support is with a QP create with max_inline_send set.
Add a common function to verify inline data support
before setting default to 64 bytes.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/openib_cma/device.c b/dapl/openib_cma/device.c
index 8cab9deb0d19835dbdf6e13ee1d8765872f3c2dd..c9bb58d813e1a40ee6807e5785bf55980ab80199 100644 (file)
--- a/dapl/openib_cma/device.c
+++ b/dapl/openib_cma/device.c
@@ -344,14 +344,16 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
        }
 
        /* set inline max with env or default, get local lid and gid 0 */
-       if (hca_ptr->ib_hca_handle->device->transport_type == IBV_TRANSPORT_IWARP)
-               hca_ptr->ib_trans.max_inline_send =
-                   dapl_os_get_env_val("DAPL_MAX_INLINE",
-                                       INLINE_SEND_IWARP_DEFAULT);
-       else
-               hca_ptr->ib_trans.max_inline_send =
-                   dapl_os_get_env_val("DAPL_MAX_INLINE",
-                                       INLINE_SEND_IB_DEFAULT);
+       if (dapl_ib_inline_data(hca_ptr->ib_hca_handle)) {
+               if (hca_ptr->ib_hca_handle->device->transport_type == IBV_TRANSPORT_IWARP)
+                       hca_ptr->ib_trans.max_inline_send =
+                           dapl_os_get_env_val("DAPL_MAX_INLINE",
+                                               INLINE_SEND_IWARP_DEFAULT);
+               else
+                       hca_ptr->ib_trans.max_inline_send =
+                           dapl_os_get_env_val("DAPL_MAX_INLINE",
+                                               INLINE_SEND_IB_DEFAULT);
+       }
 
        /* set CM timer defaults */
        hca_ptr->ib_trans.max_cm_timeout =

diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h
index e3525f24422b72bc33b95089965c6607907536d7..f2736eb367ee51787936dda30fdd1b789e45bbc5 100644 (file)
--- a/dapl/openib_common/dapl_ib_common.h
+++ b/dapl/openib_common/dapl_ib_common.h
@@ -406,6 +406,45 @@ STATIC _INLINE_ void dapl_ib_release_name (IN IB_HCA_NAME name)
        return;
 }
 
+STATIC _INLINE_ int dapl_ib_inline_data (IN struct ibv_context *ib_ctx)
+{
+       struct ibv_qp_init_attr qp_create;
+       struct ibv_pd *ib_pd = NULL;
+       struct ibv_cq *ib_cq = NULL;
+       struct ibv_qp *ib_qp = NULL;
+       int ret = 0;
+
+       ib_pd = ibv_alloc_pd(ib_ctx);
+       if (!ib_pd)
+               goto bail;
+
+       ib_cq = ibv_create_cq(ib_ctx, 10, ib_ctx, NULL, 0);
+       if (!ib_cq)
+               goto bail;
+
+       dapl_os_memzero((void *)&qp_create, sizeof(qp_create));
+       qp_create.qp_type = IBV_QPT_RC;
+       qp_create.send_cq = ib_cq;
+       qp_create.recv_cq = ib_cq;
+       qp_create.cap.max_send_wr = 1;
+       qp_create.cap.max_send_sge = 1;
+       qp_create.cap.max_inline_data = 64;
+       qp_create.qp_context = (void *)ib_ctx;
+
+       ib_qp = ibv_create_qp(ib_pd, &qp_create);
+       if (ib_qp && qp_create.cap.max_inline_data >= 64)
+               ret = 1;
+bail:
+       if (ib_qp)
+               ibv_destroy_qp(ib_qp);
+       if (ib_cq)
+               ibv_destroy_cq(ib_cq);
+       if (ib_pd)
+               ibv_dealloc_pd(ib_pd);
+
+       return ret;
+}
+
 /*
  *  Convert errno to DAT_RETURN values
  */

diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c
index 22b63c8196305028c1ff4980f24753d2f94095d0..c44f78ae903d69d5c09fb5ea32c488c4a2ae5bb1 100644 (file)
--- a/dapl/openib_common/util.c
+++ b/dapl/openib_common/util.c
@@ -555,13 +555,14 @@ skip_ib:
 
                dapl_log(DAPL_DBG_TYPE_UTIL,
                             " query_hca: (%x.%x) eps %d, sz %d evds %d,"
-                            " sz %d mtu %d - pkey %x p_idx %d sl %d gl %d\n",
+                            " sz %d mtu %d pk %x pi %d sl %d gl %d inl %d\n",
                             ia_attr->hardware_version_major,
                             ia_attr->hardware_version_minor,
                             ia_attr->max_eps, ia_attr->max_dto_per_ep,
                             ia_attr->max_evds, ia_attr->max_evd_qlen,
                             128 << tp->ib_cm.mtu, ntohs(tp->ib_cm.pkey),
-                            tp->ib_cm.pkey_idx, tp->ib_cm.sl, tp->ib_cm.global);
+                            tp->ib_cm.pkey_idx, tp->ib_cm.sl,
+                            tp->ib_cm.global, tp->ib_cm.max_inline);
 
                dapl_log(DAPL_DBG_TYPE_UTIL,
                             " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d"

diff --git a/dapl/openib_mcm/device.c b/dapl/openib_mcm/device.c
index 9d529a62e2961ced807f58a2da476e0e7fc39e03..fdf66bdc3323e9dccf3b1f46ceac151dff103890 100644 (file)
--- a/dapl/openib_mcm/device.c
+++ b/dapl/openib_mcm/device.c
@@ -255,8 +255,11 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
        }
 
        /* set RC tunables via enviroment or default */
-       hca_ptr->ib_trans.ib_cm.max_inline =
-           dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_IB_DEFAULT);
+       if (dapl_ib_inline_data(hca_ptr->ib_hca_handle)) {
+               hca_ptr->ib_trans.ib_cm.max_inline =
+                       dapl_os_get_env_val("DAPL_MAX_INLINE",
+                                           INLINE_SEND_IB_DEFAULT);
+       }
        hca_ptr->ib_trans.ib_cm.ack_retry =
            dapl_os_get_env_val("DAPL_ACK_RETRY", DCM_ACK_RETRY);
        hca_ptr->ib_trans.ib_cm.ack_timer =

diff --git a/dapl/openib_scm/device.c b/dapl/openib_scm/device.c
index 8d6cec1864c93076c88e0b062ff1ec03dbc3061b..43f9eaf7cddac1d12e7755b469d874c442e68cc0 100644 (file)
--- a/dapl/openib_scm/device.c
+++ b/dapl/openib_scm/device.c
@@ -352,8 +352,11 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
        }
 
        /* set RC tunables via enviroment or default */
-       hca_ptr->ib_trans.ib_cm.max_inline =
-           dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT);
+       if (dapl_ib_inline_data(hca_ptr->ib_hca_handle)) {
+               hca_ptr->ib_trans.ib_cm.max_inline =
+                       dapl_os_get_env_val("DAPL_MAX_INLINE",
+                                           INLINE_SEND_DEFAULT);
+       }
        hca_ptr->ib_trans.ib_cm.ack_retry =
            dapl_os_get_env_val("DAPL_ACK_RETRY", SCM_ACK_RETRY);
        hca_ptr->ib_trans.ib_cm.ack_timer =

diff --git a/dapl/openib_ucm/device.c b/dapl/openib_ucm/device.c
index 8c27f4d537d482fd7116c13a6d18667299fb02a8..b328b8796b7c685d466aa18b640565066fb7fd9b 100644 (file)
--- a/dapl/openib_ucm/device.c
+++ b/dapl/openib_ucm/device.c
@@ -273,8 +273,11 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
        }
 
        /* set RC tunables via enviroment or default */
-       hca_ptr->ib_trans.ib_cm.max_inline =
-           dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_IB_DEFAULT);
+       if (dapl_ib_inline_data(hca_ptr->ib_hca_handle)) {
+               hca_ptr->ib_trans.ib_cm.max_inline =
+                       dapl_os_get_env_val("DAPL_MAX_INLINE",
+                                            INLINE_SEND_IB_DEFAULT);
+       }
        hca_ptr->ib_trans.ib_cm.ack_retry =
            dapl_os_get_env_val("DAPL_ACK_RETRY", DCM_ACK_RETRY);
        hca_ptr->ib_trans.ib_cm.ack_timer =