[ND provider] This patch is a fix to 2333. It removes a facility to define MaxDataInl...

[ND porvider] Improved latency of ND provider by using INLINE send. [winof: 2333, 2352]

This patch adds usage of INLINE DATA facility of Mellanox HCAs for improving latency of ND provider.

Here are the ideas of the patch:
    - by default, ND provider will create QP with inline data of 160 bytes;
    (this can enlarge user's QP size)
    - one can change this default by defining environment variable IBNDPROV_MAX_INLINE_SIZE;

git-svn-id: svn://openib.tc.cornell.edu/gen1@2353 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/branches/WOF2-1/ulp/nd/user/NdEndpoint.cpp b/branches/WOF2-1/ulp/nd/user/NdEndpoint.cpp
index 3325477a8f3be2bc6b02122b9c5f9b45e41c7c30..b204320f90263d99d4491b2f6e00b1e0057ffa73 100644 (file)
--- a/branches/WOF2-1/ulp/nd/user/NdEndpoint.cpp
+++ b/branches/WOF2-1/ulp/nd/user/NdEndpoint.cpp
@@ -41,6 +41,8 @@
 #pragma warning( pop )\r
 #include "nddebug.h"\r
 \r
+extern uint32_t                        g_nd_max_inline_size;\r
+\r
 #if defined(EVENT_TRACING)\r
 #ifdef offsetof\r
 #undef offsetof\r
@@ -117,10 +119,13 @@ HRESULT CEndpoint::Initialize(
         m_pParent->m_Ifc.user_verbs.nd_get_qp_state != NULL ||\r
         m_pParent->m_Ifc.user_verbs.pre_destroy_qp != NULL ||\r
         m_pParent->m_Ifc.user_verbs.post_destroy_qp != NULL ||\r
+        m_pParent->m_Ifc.user_verbs.post_query_qp != NULL ||\r
         m_pParent->m_Ifc.user_verbs.post_send != NULL ||\r
         m_pParent->m_Ifc.user_verbs.post_recv != NULL /*||\r
         m_pParent->m_Ifc.user_verbs.bind_mw != NULL*/ );\r
 \r
+       m_MaxInlineSize = g_nd_max_inline_size;\r
+               \r
     HRESULT hr = CreateQp(\r
         pInboundCq,\r
         pOutboundCq,\r
@@ -129,11 +134,22 @@ HRESULT CEndpoint::Initialize(
         nInboundSge,\r
         nOutboundSge,\r
         InboundReadLimit,\r
-        OutboundReadLimit );\r
+        OutboundReadLimit,\r
+        m_MaxInlineSize );\r
 \r
     if( FAILED( hr ) )\r
         return hr;\r
 \r
+       ib_qp_attr_t qp_attr;\r
+       hr = QueryQp(&qp_attr);\r
+       if( FAILED( hr ) ) {\r
+               DestroyQp();\r
+               return hr;\r
+       }\r
+       else\r
+               m_MaxInlineSize = (UINT32)qp_attr.sq_max_inline;\r
+       \r
+       \r
     m_Ird = (UINT8)InboundReadLimit;\r
     m_Ord = (UINT8)OutboundReadLimit;\r
 \r
@@ -143,10 +159,7 @@ HRESULT CEndpoint::Initialize(
         DestroyQp();\r
 \r
     if( SUCCEEDED( hr ) && pMaxInlineData != NULL )\r
-    {\r
-        // Worst case.\r
-        *pMaxInlineData = nOutboundSge * 12;\r
-    }\r
+        *pMaxInlineData = m_MaxInlineSize;\r
 \r
     return hr;\r
 }\r
@@ -286,7 +299,11 @@ HRESULT CEndpoint::Send(
     wr.p_next = NULL;\r
     wr.wr_id = (ULONG_PTR)pResult;\r
     wr.wr_type = WR_SEND;\r
-    wr.send_opt = 0;\r
+    if ( pResult->BytesTransferred <= m_MaxInlineSize )\r
+           wr.send_opt = IB_SEND_OPT_INLINE; \r
+       else\r
+           wr.send_opt = 0;\r
+    \r
     if( !(Flags & ND_OP_FLAG_SILENT_SUCCESS) )\r
         wr.send_opt |= IB_SEND_OPT_SIGNALED;\r
     if( Flags & ND_OP_FLAG_READ_FENCE )\r
@@ -374,11 +391,15 @@ HRESULT CEndpoint::SendAndInvalidate(
     wr.p_next = NULL;\r
     wr.wr_id = (ULONG_PTR)pResult;\r
     wr.wr_type = WR_SEND;\r
+    if ( pResult->BytesTransferred <= m_MaxInlineSize )\r
+           wr.send_opt = IB_SEND_OPT_INLINE; \r
+       else\r
+           wr.send_opt = 0;\r
     // We simulate invalidate operations (since we simulate MW use).  We\r
     // put the RKey in the immediate data, the recipient will do the\r
     // lookup of the MW based on that (as they would with a regular\r
     // invalidate request).\r
-    wr.send_opt = IB_SEND_OPT_IMMEDIATE;\r
+    wr.send_opt |= IB_SEND_OPT_IMMEDIATE;\r
     if( !(Flags & ND_OP_FLAG_SILENT_SUCCESS) )\r
         wr.send_opt |= IB_SEND_OPT_SIGNALED;\r
     if( Flags & ND_OP_FLAG_READ_FENCE )\r
@@ -665,7 +686,10 @@ HRESULT CEndpoint::Rdma(
     wr.p_next = NULL;\r
     wr.wr_id = (ULONG_PTR)pResult;\r
     wr.wr_type = Type;\r
-    wr.send_opt = 0;\r
+    if ( pResult->BytesTransferred <= m_MaxInlineSize )\r
+           wr.send_opt = IB_SEND_OPT_INLINE; \r
+       else\r
+           wr.send_opt = 0;\r
     if( !(Flags & ND_OP_FLAG_SILENT_SUCCESS) )\r
         wr.send_opt |= IB_SEND_OPT_SIGNALED;\r
     if( Flags & ND_OP_FLAG_READ_FENCE )\r
@@ -737,11 +761,14 @@ HRESULT CEndpoint::CreateQp(
     __in SIZE_T nInboundSge,\r
     __in SIZE_T nOutboundSge,\r
     __in SIZE_T InboundReadLimit,\r
-    __in SIZE_T OutboundReadLimit\r
+    __in SIZE_T OutboundReadLimit,\r
+    __in SIZE_T MaxInlineData\r
     )\r
 {\r
     ND_ENTER( ND_DBG_NDI );\r
 \r
+    if( MaxInlineData > UINT_MAX )\r
+        return ND_INVALID_PARAMETER_3;\r
     if( nInboundEntries > UINT_MAX )\r
         return ND_INVALID_PARAMETER_4;\r
     if( nOutboundEntries > UINT_MAX )\r
@@ -764,6 +791,7 @@ HRESULT CEndpoint::CreateQp(
     qp_ioctl.in.qp_create.rq_depth = (uint32_t)nInboundEntries;\r
     qp_ioctl.in.qp_create.sq_sge = (uint32_t)nOutboundSge;\r
     qp_ioctl.in.qp_create.rq_sge = (uint32_t)nInboundSge;\r
+    qp_ioctl.in.qp_create.sq_max_inline = (uint32_t)MaxInlineData;\r
     qp_ioctl.in.qp_create.h_srq = NULL;\r
     qp_ioctl.in.qp_create.sq_signaled = FALSE;\r
 \r
@@ -941,4 +969,67 @@ HRESULT CEndpoint::ModifyQp(
     return S_OK;\r
 }\r
 \r
+HRESULT CEndpoint::QueryQp(\r
+       __out ib_qp_attr_t *qp_attr\r
+       )\r
+{\r
+       ib_api_status_t status;\r
+       \r
+       ND_ENTER( ND_DBG_NDI );\r
+\r
+       ual_query_qp_ioctl_t qp_ioctl;\r
+       cl_memclr( &qp_ioctl, sizeof(qp_ioctl) );\r
+       qp_ioctl.in.h_qp = m_hQp;\r
+\r
+       /* Call the uvp pre call if the vendor library provided a valid ca handle */\r
+       if( m_pParent->m_Ifc.user_verbs.pre_query_qp )\r
+       {\r
+               /* Pre call to the UVP library */\r
+               status = m_pParent->m_Ifc.user_verbs.pre_query_qp( m_uQp, &qp_ioctl.in.umv_buf );\r
+               if( status != IB_SUCCESS )\r
+                       goto done;\r
+       }\r
+\r
+       DWORD bytes_ret;\r
+       BOOL fSuccess = DeviceIoControl(\r
+               m_pParent->m_hSync,\r
+               UAL_QUERY_QP,\r
+               &qp_ioctl.in,\r
+               sizeof(qp_ioctl.in),\r
+               &qp_ioctl.out,\r
+               sizeof(qp_ioctl.out),\r
+               &bytes_ret,\r
+               NULL\r
+               );\r
+\r
+       if( fSuccess != TRUE || bytes_ret != sizeof(qp_ioctl.out) )\r
+               status = IB_ERROR;\r
+       else\r
+               status = qp_ioctl.out.status;\r
+\r
+       /* Call vendor's post_query_qp */\r
+       CL_ASSERT( m_pParent->m_Ifc.user_verbs.post_query_qp );\r
+       if( m_pParent->m_Ifc.user_verbs.post_query_qp )\r
+       {\r
+               m_pParent->m_Ifc.user_verbs.post_query_qp( m_uQp, status,\r
+                       &qp_ioctl.out.attr, &qp_ioctl.out.umv_buf );\r
+       }\r
+\r
+done:\r
+       ND_PRINT( TRACE_LEVEL_INFORMATION, ND_DBG_NDI,\r
+               ("Queried QP %#I64x, QPn %#x, pd %#I64x, context %p, status %#x \n", \r
+               m_hQp, m_Qpn, m_pParent->m_hPd, this, status ) );\r
+\r
+       switch( status )\r
+       {\r
+       case IB_SUCCESS:\r
+               *qp_attr = qp_ioctl.out.attr;\r
+               return S_OK;\r
+\r
+       default:\r
+               return ND_UNSUCCESSFUL;\r
+       }\r
+       \r
+}\r
+\r
 } // namespace\r

diff --git a/branches/WOF2-1/ulp/nd/user/NdEndpoint.h b/branches/WOF2-1/ulp/nd/user/NdEndpoint.h
index f47a6c34818a03fcda102fd2693b3c06d24f4e46..fd1eabd4824ec63be2411a8f4803922a9d2b4cc0 100644 (file)
--- a/branches/WOF2-1/ulp/nd/user/NdEndpoint.h
+++ b/branches/WOF2-1/ulp/nd/user/NdEndpoint.h
@@ -176,11 +176,16 @@ private:
         __in SIZE_T nInboundSge,\r
         __in SIZE_T nOutboundSge,\r
         __in SIZE_T InboundReadLimit,\r
-        __in SIZE_T OutboundReadLimit\r
+               __in SIZE_T OutboundReadLimit,\r
+               __in SIZE_T MaxInlineData\r
         );\r
 \r
     void DestroyQp();\r
 \r
+    HRESULT QueryQp(\r
+               __out ib_qp_attr_t *qp_attr\r
+               );\r
+\r
     HRESULT ModifyQp(\r
         __in ib_qp_state_t NewState\r
         );\r
@@ -197,6 +202,7 @@ protected:
 \r
     UINT8 m_Ird;\r
     UINT8 m_Ord;\r
+       UINT32 m_MaxInlineSize;\r
 };\r
 \r
 } // namespace\r

diff --git a/branches/WOF2-1/ulp/nd/user/NdProv.cpp b/branches/WOF2-1/ulp/nd/user/NdProv.cpp
index 720ef076d08b61ec93c125baad4ebac38abf02a1..fe584baabdc1f33b7f5190b966fc2c0f2b253689 100644 (file)
--- a/branches/WOF2-1/ulp/nd/user/NdProv.cpp
+++ b/branches/WOF2-1/ulp/nd/user/NdProv.cpp
@@ -57,6 +57,7 @@
 uint32_t g_nd_dbg_level = TRACE_LEVEL_ERROR;\r
 /* WPP doesn't want here literals! */\r
 uint32_t g_nd_dbg_flags = 0x80000001; /* ND_DBG_ERROR | ND_DBG_NDI; */\r
+uint32_t g_nd_max_inline_size = 160;\r
 \r
 HANDLE ghHeap;\r
 \r
@@ -462,6 +463,8 @@ _DllMain(
     switch( dwReason )\r
     {\r
     case DLL_PROCESS_ATTACH:\r
+        TCHAR    env_var[16];\r
+        DWORD    i;\r
 \r
 \r
 #if defined(EVENT_TRACING)\r
@@ -471,9 +474,6 @@ _DllMain(
         WPP_INIT_TRACING(L"ibndprov.dll");\r
 #endif\r
 #elif DBG \r
-        TCHAR    env_var[16];\r
-        DWORD    i;\r
-\r
         i = GetEnvironmentVariable( "IBNDPROV_DBG_LEVEL", env_var, sizeof(env_var) );\r
         if( i && i <= 16 )\r
         {\r
@@ -494,6 +494,12 @@ _DllMain(
             GetCurrentProcessId(), g_nd_dbg_level ,g_nd_dbg_flags) );\r
 #endif\r
 \r
+               i = GetEnvironmentVariable( "IBNDPROV_MAX_INLINE_SIZE", env_var, sizeof(env_var) );\r
+               if( i && i <= 16 )\r
+               {\r
+                       g_nd_max_inline_size = _tcstoul( env_var, NULL, 16 );\r
+               }\r
+\r
         ghHeap = HeapCreate( 0, 0, 0 );\r
         if( ghHeap == NULL )\r
         {\r