From df809975ebe3a8e0d9d6deef15c347c0227d40ea Mon Sep 17 00:00:00 2001 From: Stan Smith Date: Wed, 13 Jan 2010 19:04:32 +0000 Subject: [PATCH] [NETDIRECT] [PATCH 4/4] winverbs/netdirect: add support for inline sends By default, use inline sends, and allow user to limit inline send size through an environment variable. To support existing deployments, winverbs make use of the same environment setting as the ibal ND provider. Signed-off-by: Sean Hefty --- Note: The kernel HCA drivers report 0 for the max inline send value for user space QPs, even if inline sends are enabled. I didn't understand how to fix the kernel drivers, so they could report the correct value. The winverbs ND code simply returns the value that it uses when creating the QP. trunk/ulp/netdirect/user/nd_ep.cpp trunk/ulp/netdirect/user/nd_ep.h git-svn-id: svn://openib.tc.cornell.edu/gen1@2660 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- branches/WOF2-2/ulp/netdirect/user/nd_ep.cpp | 23 ++++++++++++++++---- branches/WOF2-2/ulp/netdirect/user/nd_ep.h | 2 ++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/branches/WOF2-2/ulp/netdirect/user/nd_ep.cpp b/branches/WOF2-2/ulp/netdirect/user/nd_ep.cpp index d0fe6ea3..7060b208 100644 --- a/branches/WOF2-2/ulp/netdirect/user/nd_ep.cpp +++ b/branches/WOF2-2/ulp/netdirect/user/nd_ep.cpp @@ -44,6 +44,16 @@ CNDEndpoint::CNDEndpoint(CNDConnector *pConnector) m_pOutboundCq = NULL; } +STDMETHODIMP_(void) CNDEndpoint:: +InitMaxInline(void) +{ + TCHAR val[16]; + DWORD ret; + + ret = GetEnvironmentVariable("IBNDPROV_MAX_INLINE_SIZE", val, 16); + m_MaxInlineSend = (ret > 0 && ret <= 16) ? (SIZE_T) strtoul(val, NULL, 16) : 160; +} + STDMETHODIMP CNDEndpoint:: Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq, SIZE_T nInboundEntries, SIZE_T nOutboundEntries, @@ -63,6 +73,7 @@ Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq, m_pOutboundCq->AddRef(); m_InitiatorDepth = OutboundReadLimit; m_ResponderResources = InboundReadLimit; + InitMaxInline(); RtlZeroMemory(&create, sizeof create); create.pSendCq = pOutboundCq->m_pWvCq; @@ -74,6 +85,7 @@ Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq, create.ReceiveSge = nInboundSge; create.InitiatorDepth = OutboundReadLimit; create.ResponderResources = InboundReadLimit; + create.MaxInlineSend = m_MaxInlineSend; create.QpType = WvQpTypeRc; hr = m_pConnector->m_pAdapter->m_pWvPd->CreateConnectQueuePair(&create, &m_pWvQp); @@ -97,7 +109,7 @@ Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq, } if (pMaxInlineData) { - *pMaxInlineData = 0; + *pMaxInlineData = m_MaxInlineSend; } return ND_SUCCESS; } @@ -212,7 +224,8 @@ Send(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge, DWORD Flags) HRESULT hr; pResult->BytesTransferred = ConvertSgl(pSgl, nSge, sgl); - opts = ConvertSendFlags(Flags) | (pSgl && pSgl[0].hMr ? 0 : WV_SEND_INLINE); + opts = ConvertSendFlags(Flags) | + (pResult->BytesTransferred <= m_MaxInlineSend ? WV_SEND_INLINE : 0); hr = m_pWvQp->Send((UINT64) pResult, sgl, nSge, opts, 0); return NDConvertWVStatus(hr); } @@ -278,7 +291,8 @@ Read(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge, HRESULT hr; pResult->BytesTransferred = ConvertSgl(pSgl, nSge, sgl); - opts = ConvertSendFlags(Flags) | (pSgl[0].hMr ? 0 : WV_SEND_INLINE); + opts = ConvertSendFlags(Flags) | + (pResult->BytesTransferred ? 0 : WV_SEND_INLINE); addr = ntohll(pRemoteMwDescriptor->Base) + Offset; hr = m_pWvQp->Read((UINT64) pResult, sgl, nSge, opts, htonll(addr), pRemoteMwDescriptor->Token); @@ -295,7 +309,8 @@ Write(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge, HRESULT hr; pResult->BytesTransferred = ConvertSgl(pSgl, nSge, sgl); - opts = ConvertSendFlags(Flags) | (pSgl && pSgl[0].hMr ? 0 : WV_SEND_INLINE); + opts = ConvertSendFlags(Flags) | + (pResult->BytesTransferred <= m_MaxInlineSend ? WV_SEND_INLINE : 0); addr = ntohll(pRemoteMwDescriptor->Base) + Offset; hr = m_pWvQp->Write((UINT64) pResult, sgl, nSge, opts, 0, htonll(addr), pRemoteMwDescriptor->Token); diff --git a/branches/WOF2-2/ulp/netdirect/user/nd_ep.h b/branches/WOF2-2/ulp/netdirect/user/nd_ep.h index a49c43e5..78fac3fa 100644 --- a/branches/WOF2-2/ulp/netdirect/user/nd_ep.h +++ b/branches/WOF2-2/ulp/netdirect/user/nd_ep.h @@ -111,6 +111,7 @@ public: IWVConnectQueuePair *m_pWvQp; SIZE_T m_InitiatorDepth; SIZE_T m_ResponderResources; + SIZE_T m_MaxInlineSend; protected: CNDConnector *m_pConnector; @@ -122,6 +123,7 @@ protected: SIZE_T nInboundSge, SIZE_T nOutboundSge, SIZE_T InboundReadLimit, SIZE_T OutboundReadLimit, SIZE_T* pMaxInlineData); + STDMETHODIMP_(void) InitMaxInline(); STDMETHODIMP_(SIZE_T) ConvertSgl(const ND_SGE* pSgl, SIZE_T nSge, WV_SGE *pWvSgl); STDMETHODIMP_(DWORD) ConvertSendFlags(DWORD Flags); STDMETHODIMP_(DWORD) ConvertAccessFlags(DWORD Flags); -- 2.46.0