From 54c037af69cca45e111955dfa42a3bcdb2b553de Mon Sep 17 00:00:00 2001 From: tzachid Date: Mon, 21 Nov 2005 12:12:24 +0000 Subject: [PATCH] Moved send to register and use physical memory (Rev 80) git-svn-id: svn://openib.tc.cornell.edu/gen1@165 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/ulp/sdp/kernel/SdpBufferPool.cpp | 26 ++++------- trunk/ulp/sdp/kernel/SdpBufferPool.h | 4 +- trunk/ulp/sdp/kernel/SdpSocket.cpp | 60 +++++++++++++++++++++++--- trunk/ulp/sdp/kernel/SdpSocket.h | 4 ++ trunk/ulp/sdp/todo | 21 +++++++++ 5 files changed, 91 insertions(+), 24 deletions(-) create mode 100644 trunk/ulp/sdp/todo diff --git a/trunk/ulp/sdp/kernel/SdpBufferPool.cpp b/trunk/ulp/sdp/kernel/SdpBufferPool.cpp index b71a6be2..e801a4dc 100644 --- a/trunk/ulp/sdp/kernel/SdpBufferPool.cpp +++ b/trunk/ulp/sdp/kernel/SdpBufferPool.cpp @@ -9,7 +9,8 @@ BufferPool::Init( int MaxConcurrentSends, int MaxMessageSize, ib_pd_handle_t pd, - ib_qp_handle_t qp + ib_qp_handle_t qp, + net32_t lkey ) { SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); @@ -25,6 +26,8 @@ BufferPool::Init( m_pd = pd; ASSERT(qp != NULL); m_qp = qp; + ASSERT(lkey != NULL); + m_lkey = lkey; return STATUS_SUCCESS; } @@ -206,8 +209,6 @@ BufferPool::AllocateBuffer(BufferDescriptor ** ppBufferDescriptor) SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); NTSTATUS rc = STATUS_SUCCESS; BufferDescriptor *pBufferDescriptor = NULL; - ib_mr_create_t mr_create; - uint32_t rkey; // Allocate the buffer descriptor pBufferDescriptor = @@ -240,18 +241,7 @@ BufferPool::AllocateBuffer(BufferDescriptor ** ppBufferDescriptor) pBufferDescriptor->BufferSize = m_MaxMessageSize; pBufferDescriptor->DataSize = 0; pBufferDescriptor->mr_handle = NULL; - - // Now we need to register this memory with the hardware - mr_create.vaddr = pBufferDescriptor->pBuffer; - mr_create.length = pBufferDescriptor->BufferSize; - mr_create.access_ctrl = IB_AC_LOCAL_WRITE; - ib_api_status_t ib_status = ib_reg_mem( m_pd, &mr_create, &pBufferDescriptor->ds_array.lkey, &rkey, &pBufferDescriptor->mr_handle ); - if( ib_status != IB_SUCCESS ) { - SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("ib_reg_mem failed ib_status = 0x%d\n", ib_status )); - rc = IB2Status(ib_status); - goto Cleanup; - } Cleanup: if (!NT_SUCCESS(rc)) { @@ -269,9 +259,8 @@ Cleanup: VOID BufferPool::DeAllocateBuffer(BufferDescriptor *pBufferDescriptor) { - //????? clear the memory here. - // ????? - + ExFreePoolWithTag(pBufferDescriptor->pBuffer, SEND_BUFFERS_ALLOCATION_TAG); + ExFreePoolWithTag(pBufferDescriptor, SEND_BUFFERS_ALLOCATION_TAG); } NTSTATUS @@ -301,7 +290,8 @@ BufferPool::SendBuffer(BufferDescriptor *pBufferDescriptor) send_wr.send_opt = IB_SEND_OPT_SIGNALED;//socket_info->send_opt; pBufferDescriptor->ds_array.length = pBufferDescriptor->DataSize + sizeof msg_hdr_bsdh; - pBufferDescriptor->ds_array.vaddr = (uint64_t)(void* __ptr64) pBufferDescriptor->pBuffer; + pBufferDescriptor->ds_array.vaddr = MmGetPhysicalAddress( pBufferDescriptor->pBuffer ).QuadPart; + pBufferDescriptor->ds_array.lkey = m_lkey; send_wr.num_ds = 1; send_wr.ds_array = &pBufferDescriptor->ds_array; diff --git a/trunk/ulp/sdp/kernel/SdpBufferPool.h b/trunk/ulp/sdp/kernel/SdpBufferPool.h index a88fd35d..a73e6464 100644 --- a/trunk/ulp/sdp/kernel/SdpBufferPool.h +++ b/trunk/ulp/sdp/kernel/SdpBufferPool.h @@ -77,7 +77,8 @@ public: int MaxConcurrentSends, int MaxMessageSize, ib_pd_handle_t pd, - ib_qp_handle_t qp + ib_qp_handle_t qp, + net32_t lkey ); NTSTATUS GetBuffer( @@ -124,6 +125,7 @@ private: // TODO: Should they stay here and be used like this ? ib_pd_handle_t m_pd; ib_qp_handle_t m_qp; + net32_t m_lkey; // A list of events that the users has to wait on. ???? currently only one KEVENT m_WaitingClients; // switch to a linked list diff --git a/trunk/ulp/sdp/kernel/SdpSocket.cpp b/trunk/ulp/sdp/kernel/SdpSocket.cpp index a1853604..807eee60 100644 --- a/trunk/ulp/sdp/kernel/SdpSocket.cpp +++ b/trunk/ulp/sdp/kernel/SdpSocket.cpp @@ -70,6 +70,7 @@ SdpSocket::SdpSocket() m_rcq = NULL; m_scq = NULL; m_qp = NULL; + m_mr = NULL; m_state = SS_IDLE; } @@ -90,6 +91,7 @@ NTSTATUS SdpSocket::Init( return rc; } +# if 0 struct sdpc_buff { // struct sdpc_buff *next; // struct sdpc_buff *prev; @@ -130,6 +132,7 @@ const int BUFFER_SIZE = 4000 + 16;//65536; #define SDP_BUFF_F_SET_UNSIG(buff) ((buff)->flags |= SDP_BUFF_F_UNSIG) #define SDP_BUFF_F_CLR_UNSIG(buff) ((buff)->flags &= (~SDP_BUFF_F_UNSIG)) +#endif NTSTATUS SdpSocket::WSPSend( WspSendIn *pWspSendIn, @@ -605,7 +608,7 @@ NTSTATUS SdpSocket::CmSendRTU() int MaxMessageSize = min(m_hello_ack.hah.l_rcv_size, MAX_SEND_BUFFER_SIZE); - rc = m_SendBufferPool.Init(MAX_SEND_PACKETS, QP_ATTRIB_SQ_DEPTH, MaxMessageSize, m_pd, m_qp); + rc = m_SendBufferPool.Init(MAX_SEND_PACKETS, QP_ATTRIB_SQ_DEPTH, MaxMessageSize, m_pd, m_qp, m_lkey); if (!NT_SUCCESS(rc)) { SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_SendBufferPool.Init failed rc = 0x%x\n", rc )); goto Cleanup; @@ -841,9 +844,14 @@ __qp_event1( NTSTATUS SdpSocket::CreateQp() { NTSTATUS rc = STATUS_SUCCESS; - ib_cq_create_t cq_create; - ib_qp_create_t qp_create; - ib_api_status_t ib_status; + ib_cq_create_t cq_create; + ib_qp_create_t qp_create; + ib_api_status_t ib_status; + ib_phys_create_t phys_create; + ib_phys_range_t phys_range; + uint64_t vaddr; + net32_t rkey; + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("CreateQp called this = 0x%p\n", this)); /* Open the CA. */ @@ -941,9 +949,35 @@ NTSTATUS SdpSocket::CreateQp() return status; } p_port->ib_mgr.qpn = qp_attr.num; -#endif +#endif + const net64_t MEM_REG_SIZE = 0xFFFFFFFFFFFFFFFF; + /* Register all of physical memory */ + phys_create.length = MEM_REG_SIZE; + phys_create.num_ranges = 1; + phys_create.range_array = &phys_range; + phys_create.buf_offset = 0; + phys_create.hca_page_size = PAGE_SIZE; + phys_create.access_ctrl = IB_AC_LOCAL_WRITE; + phys_range.base_addr = 0; + phys_range.size = MEM_REG_SIZE; + vaddr = 0; + ib_status = ib_reg_phys( + m_pd, + &phys_create, + &vaddr, + &m_lkey, + &rkey, + &m_mr ); + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_reg_phys failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto Cleanup; + } Cleanup: + // In the case of failure the mess will be cleaned on + // shutdown + // TODO: Need to make a beter cleanup here return rc; } @@ -1037,6 +1071,8 @@ VOID SdpSocket::Shutdown() { //???? locking // if(m_shutdown - on the lock) ??? + ib_api_status_t ib_status; + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Shutdown called this = 0x%p\n", this)); @@ -1062,6 +1098,20 @@ VOID SdpSocket::Shutdown() // Now that all ibal operations have finished we can free the memory m_SendBufferPool.ShutDown(); + + +/* + Memory reagion probably cleans when the other handles are closed + if (m_mr != NULL) { + ib_status = ib_dereg_mr(m_mr); + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_dereg_mr failed ib_status = 0x%d\n", ib_status )); + ASSERT( ib_status == IB_SUCCESS ); + } + m_mr = NULL; + } +*/ + } /* diff --git a/trunk/ulp/sdp/kernel/SdpSocket.h b/trunk/ulp/sdp/kernel/SdpSocket.h index adb4420c..97ffdea3 100644 --- a/trunk/ulp/sdp/kernel/SdpSocket.h +++ b/trunk/ulp/sdp/kernel/SdpSocket.h @@ -66,6 +66,10 @@ private: ib_qp_handle_t m_qp; + ib_mr_handle_t m_mr; + + net32_t m_lkey; + BufferPool m_SendBufferPool; KEVENT m_ConnectCmCompleteEvent; diff --git a/trunk/ulp/sdp/todo b/trunk/ulp/sdp/todo new file mode 100644 index 00000000..3fcf50d1 --- /dev/null +++ b/trunk/ulp/sdp/todo @@ -0,0 +1,21 @@ +This file includes things that should be impreoved in the sdp implmentation +============================================================================= + +KERNEL MODE: +1) On send: implmeant some kind of a negal algorithm. +2) On send: Create some kind of mechanism that will allow to recieve complitions on more than +one send. + + + + +USER MODE: + + +* Check the lifetime of the SdpSocket (when is it deleted and so)?? + +* make sure that the asserts are implmented in debug +* check with intel that we can remove their lisence from the files. + + +* make sure that the SDP socket is deleted at least in the main path \ No newline at end of file -- 2.41.0