From 63818c704511970b84dd8285a2fefbb1ef498fe0 Mon Sep 17 00:00:00 2001 From: leonidk Date: Thu, 6 Aug 2009 17:13:25 +0000 Subject: [PATCH] [ND provider] improved latency of ND provider by using INLINE send This patch adds usage of INLINE DATA facility of Mellanox HCAs for improving latency of ND provider. Here are the ideas of the patch: - by default, ND provider will create QP with inline data of 160 bytes; (this can enlarge user's QP size) - one can change this default by defining environment variable IBNDPROV_MAX_INLINE_SIZE; - an ND application, while creating QP, can define the necessary INLINE DATA size; this value takes precedence over the default one. git-svn-id: svn://openib.tc.cornell.edu/gen1@2333 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/ulp/nd/user/NdEndpoint.cpp | 116 ++++++++++++++++++++++++++++--- trunk/ulp/nd/user/NdEndpoint.h | 10 ++- trunk/ulp/nd/user/NdProv.cpp | 12 +++- 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/trunk/ulp/nd/user/NdEndpoint.cpp b/trunk/ulp/nd/user/NdEndpoint.cpp index 3325477a..5d0cee13 100644 --- a/trunk/ulp/nd/user/NdEndpoint.cpp +++ b/trunk/ulp/nd/user/NdEndpoint.cpp @@ -41,6 +41,8 @@ #pragma warning( pop ) #include "nddebug.h" +extern uint32_t g_nd_max_inline_size; + #if defined(EVENT_TRACING) #ifdef offsetof #undef offsetof @@ -96,7 +98,7 @@ HRESULT CEndpoint::Initialize( __in SIZE_T nOutboundSge, __in SIZE_T InboundReadLimit, __in SIZE_T OutboundReadLimit, - __out_opt SIZE_T* pMaxInlineData + __in_opt __out_opt SIZE_T* pMaxInlineData ) { ND_ENTER( ND_DBG_NDI ); @@ -117,10 +119,17 @@ HRESULT CEndpoint::Initialize( m_pParent->m_Ifc.user_verbs.nd_get_qp_state != NULL || m_pParent->m_Ifc.user_verbs.pre_destroy_qp != NULL || m_pParent->m_Ifc.user_verbs.post_destroy_qp != NULL || + m_pParent->m_Ifc.user_verbs.post_query_qp != NULL || m_pParent->m_Ifc.user_verbs.post_send != NULL || m_pParent->m_Ifc.user_verbs.post_recv != NULL /*|| m_pParent->m_Ifc.user_verbs.bind_mw != NULL*/ ); + UINT32 InlineSize; + if ( pMaxInlineData ) + InlineSize = (UINT32)*pMaxInlineData; + else + InlineSize = g_nd_max_inline_size; + HRESULT hr = CreateQp( pInboundCq, pOutboundCq, @@ -129,13 +138,25 @@ HRESULT CEndpoint::Initialize( nInboundSge, nOutboundSge, InboundReadLimit, - OutboundReadLimit ); + OutboundReadLimit, + InlineSize ); if( FAILED( hr ) ) return hr; + ib_qp_attr_t qp_attr; + hr = QueryQp(&qp_attr); + if( FAILED( hr ) ) { + DestroyQp(); + return hr; + } + else + InlineSize = (UINT32)qp_attr.sq_max_inline; + + m_Ird = (UINT8)InboundReadLimit; m_Ord = (UINT8)OutboundReadLimit; + m_MaxInlineSize = InlineSize; // Move the QP to the INIT state so users can post receives. hr = ModifyQp( IB_QPS_INIT ); @@ -143,10 +164,7 @@ HRESULT CEndpoint::Initialize( DestroyQp(); if( SUCCEEDED( hr ) && pMaxInlineData != NULL ) - { - // Worst case. - *pMaxInlineData = nOutboundSge * 12; - } + *pMaxInlineData = InlineSize; return hr; } @@ -286,7 +304,11 @@ HRESULT CEndpoint::Send( wr.p_next = NULL; wr.wr_id = (ULONG_PTR)pResult; wr.wr_type = WR_SEND; - wr.send_opt = 0; + if ( pResult->BytesTransferred <= m_MaxInlineSize ) + wr.send_opt = IB_SEND_OPT_INLINE; + else + wr.send_opt = 0; + if( !(Flags & ND_OP_FLAG_SILENT_SUCCESS) ) wr.send_opt |= IB_SEND_OPT_SIGNALED; if( Flags & ND_OP_FLAG_READ_FENCE ) @@ -374,11 +396,15 @@ HRESULT CEndpoint::SendAndInvalidate( wr.p_next = NULL; wr.wr_id = (ULONG_PTR)pResult; wr.wr_type = WR_SEND; + if ( pResult->BytesTransferred <= m_MaxInlineSize ) + wr.send_opt = IB_SEND_OPT_INLINE; + else + wr.send_opt = 0; // We simulate invalidate operations (since we simulate MW use). We // put the RKey in the immediate data, the recipient will do the // lookup of the MW based on that (as they would with a regular // invalidate request). - wr.send_opt = IB_SEND_OPT_IMMEDIATE; + wr.send_opt |= IB_SEND_OPT_IMMEDIATE; if( !(Flags & ND_OP_FLAG_SILENT_SUCCESS) ) wr.send_opt |= IB_SEND_OPT_SIGNALED; if( Flags & ND_OP_FLAG_READ_FENCE ) @@ -665,7 +691,10 @@ HRESULT CEndpoint::Rdma( wr.p_next = NULL; wr.wr_id = (ULONG_PTR)pResult; wr.wr_type = Type; - wr.send_opt = 0; + if ( pResult->BytesTransferred <= m_MaxInlineSize ) + wr.send_opt = IB_SEND_OPT_INLINE; + else + wr.send_opt = 0; if( !(Flags & ND_OP_FLAG_SILENT_SUCCESS) ) wr.send_opt |= IB_SEND_OPT_SIGNALED; if( Flags & ND_OP_FLAG_READ_FENCE ) @@ -737,11 +766,14 @@ HRESULT CEndpoint::CreateQp( __in SIZE_T nInboundSge, __in SIZE_T nOutboundSge, __in SIZE_T InboundReadLimit, - __in SIZE_T OutboundReadLimit + __in SIZE_T OutboundReadLimit, + __in SIZE_T MaxInlineData ) { ND_ENTER( ND_DBG_NDI ); + if( MaxInlineData > UINT_MAX ) + return ND_INVALID_PARAMETER_3; if( nInboundEntries > UINT_MAX ) return ND_INVALID_PARAMETER_4; if( nOutboundEntries > UINT_MAX ) @@ -764,6 +796,7 @@ HRESULT CEndpoint::CreateQp( qp_ioctl.in.qp_create.rq_depth = (uint32_t)nInboundEntries; qp_ioctl.in.qp_create.sq_sge = (uint32_t)nOutboundSge; qp_ioctl.in.qp_create.rq_sge = (uint32_t)nInboundSge; + qp_ioctl.in.qp_create.sq_max_inline = (uint32_t)MaxInlineData; qp_ioctl.in.qp_create.h_srq = NULL; qp_ioctl.in.qp_create.sq_signaled = FALSE; @@ -941,4 +974,67 @@ HRESULT CEndpoint::ModifyQp( return S_OK; } +HRESULT CEndpoint::QueryQp( + __out ib_qp_attr_t *qp_attr + ) +{ + ib_api_status_t status; + + ND_ENTER( ND_DBG_NDI ); + + ual_query_qp_ioctl_t qp_ioctl; + cl_memclr( &qp_ioctl, sizeof(qp_ioctl) ); + qp_ioctl.in.h_qp = m_hQp; + + /* Call the uvp pre call if the vendor library provided a valid ca handle */ + if( m_pParent->m_Ifc.user_verbs.pre_query_qp ) + { + /* Pre call to the UVP library */ + status = m_pParent->m_Ifc.user_verbs.pre_query_qp( m_uQp, &qp_ioctl.in.umv_buf ); + if( status != IB_SUCCESS ) + goto done; + } + + DWORD bytes_ret; + BOOL fSuccess = DeviceIoControl( + m_pParent->m_hSync, + UAL_QUERY_QP, + &qp_ioctl.in, + sizeof(qp_ioctl.in), + &qp_ioctl.out, + sizeof(qp_ioctl.out), + &bytes_ret, + NULL + ); + + if( fSuccess != TRUE || bytes_ret != sizeof(qp_ioctl.out) ) + status = IB_ERROR; + else + status = qp_ioctl.out.status; + + /* Call vendor's post_query_qp */ + CL_ASSERT( m_pParent->m_Ifc.user_verbs.post_query_qp ); + if( m_pParent->m_Ifc.user_verbs.post_query_qp ) + { + m_pParent->m_Ifc.user_verbs.post_query_qp( m_uQp, status, + &qp_ioctl.out.attr, &qp_ioctl.out.umv_buf ); + } + +done: + ND_PRINT( TRACE_LEVEL_INFORMATION, ND_DBG_NDI, + ("Queried QP %#I64x, QPn %#x, pd %#I64x, context %p, status %#x \n", + m_hQp, m_Qpn, m_pParent->m_hPd, this, status ) ); + + switch( status ) + { + case IB_SUCCESS: + *qp_attr = qp_ioctl.out.attr; + return S_OK; + + default: + return ND_UNSUCCESSFUL; + } + +} + } // namespace diff --git a/trunk/ulp/nd/user/NdEndpoint.h b/trunk/ulp/nd/user/NdEndpoint.h index f47a6c34..f72bdb95 100644 --- a/trunk/ulp/nd/user/NdEndpoint.h +++ b/trunk/ulp/nd/user/NdEndpoint.h @@ -67,7 +67,7 @@ private: __in SIZE_T nOutboundSge, __in SIZE_T InboundReadLimit, __in SIZE_T OutboundReadLimit, - __out_opt SIZE_T* pMaxInlineData + __in_opt __out_opt SIZE_T* pMaxInlineData ); public: @@ -176,11 +176,16 @@ private: __in SIZE_T nInboundSge, __in SIZE_T nOutboundSge, __in SIZE_T InboundReadLimit, - __in SIZE_T OutboundReadLimit + __in SIZE_T OutboundReadLimit, + __in SIZE_T MaxInlineData ); void DestroyQp(); + HRESULT QueryQp( + __out ib_qp_attr_t *qp_attr + ); + HRESULT ModifyQp( __in ib_qp_state_t NewState ); @@ -197,6 +202,7 @@ protected: UINT8 m_Ird; UINT8 m_Ord; + UINT32 m_MaxInlineSize; }; } // namespace diff --git a/trunk/ulp/nd/user/NdProv.cpp b/trunk/ulp/nd/user/NdProv.cpp index 720ef076..fe584baa 100644 --- a/trunk/ulp/nd/user/NdProv.cpp +++ b/trunk/ulp/nd/user/NdProv.cpp @@ -57,6 +57,7 @@ uint32_t g_nd_dbg_level = TRACE_LEVEL_ERROR; /* WPP doesn't want here literals! */ uint32_t g_nd_dbg_flags = 0x80000001; /* ND_DBG_ERROR | ND_DBG_NDI; */ +uint32_t g_nd_max_inline_size = 160; HANDLE ghHeap; @@ -462,6 +463,8 @@ _DllMain( switch( dwReason ) { case DLL_PROCESS_ATTACH: + TCHAR env_var[16]; + DWORD i; #if defined(EVENT_TRACING) @@ -471,9 +474,6 @@ _DllMain( WPP_INIT_TRACING(L"ibndprov.dll"); #endif #elif DBG - TCHAR env_var[16]; - DWORD i; - i = GetEnvironmentVariable( "IBNDPROV_DBG_LEVEL", env_var, sizeof(env_var) ); if( i && i <= 16 ) { @@ -494,6 +494,12 @@ _DllMain( GetCurrentProcessId(), g_nd_dbg_level ,g_nd_dbg_flags) ); #endif + i = GetEnvironmentVariable( "IBNDPROV_MAX_INLINE_SIZE", env_var, sizeof(env_var) ); + if( i && i <= 16 ) + { + g_nd_max_inline_size = _tcstoul( env_var, NULL, 16 ); + } + ghHeap = HeapCreate( 0, 0, 0 ); if( ghHeap == NULL ) { -- 2.41.0