From 556ca56ba118274d73c0aa44f5f88e8e5d71f7af Mon Sep 17 00:00:00 2001 From: tzachid Date: Mon, 21 Nov 2005 12:22:16 +0000 Subject: [PATCH] CloseSocket is now implemented (needs some more tuning) (Rev 315) git-svn-id: svn://openib.tc.cornell.edu/gen1@173 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/ulp/sdp/include/SdpShared.h | 25 +- trunk/ulp/sdp/kernel/SdpArp.h | 42 ++ trunk/ulp/sdp/kernel/SdpBufferPool.cpp | 109 ++++- trunk/ulp/sdp/kernel/SdpBufferPool.h | 10 + trunk/ulp/sdp/kernel/SdpConnectionList.cpp | 23 + trunk/ulp/sdp/kernel/SdpConnectionList.h | 3 + trunk/ulp/sdp/kernel/SdpDriver.cpp | 44 +- trunk/ulp/sdp/kernel/SdpGenUtils.cpp | 34 +- trunk/ulp/sdp/kernel/SdpGenUtils.h | 5 + trunk/ulp/sdp/kernel/SdpLock.h | 2 - trunk/ulp/sdp/kernel/SdpRecvPool.cpp | 89 +++- trunk/ulp/sdp/kernel/SdpRecvPool.h | 7 + trunk/ulp/sdp/kernel/SdpSocket.cpp | 483 +++++++++++++++++---- trunk/ulp/sdp/kernel/SdpSocket.h | 55 ++- trunk/ulp/sdp/kernel/SdpTrace.cpp | 3 +- trunk/ulp/sdp/kernel/SdpTrace.h | 2 +- trunk/ulp/sdp/kernel/SdpUserFile.cpp | 20 + trunk/ulp/sdp/kernel/SdpUserFile.h | 3 +- trunk/ulp/sdp/todo | 6 +- 19 files changed, 820 insertions(+), 145 deletions(-) diff --git a/trunk/ulp/sdp/include/SdpShared.h b/trunk/ulp/sdp/include/SdpShared.h index f647de4e..6638927d 100644 --- a/trunk/ulp/sdp/include/SdpShared.h +++ b/trunk/ulp/sdp/include/SdpShared.h @@ -11,14 +11,14 @@ // Define the IOCTL codes that will be used for sending the requests down -#define IOCTL_WSP_SOCKET CTL_CODE(FILE_DEVICE_UNKNOWN, 0x801, METHOD_BUFFERED ,FILE_ANY_ACCESS) -#define IOCTL_WSP_CONNECT CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED ,FILE_ANY_ACCESS) -#define IOCTL_WSP_SEND CTL_CODE(FILE_DEVICE_UNKNOWN, 0x803, METHOD_BUFFERED ,FILE_ANY_ACCESS) -#define IOCTL_WSP_RECV CTL_CODE(FILE_DEVICE_UNKNOWN, 0x804, METHOD_BUFFERED ,FILE_ANY_ACCESS) -#define IOCTL_WSP_BIND CTL_CODE(FILE_DEVICE_UNKNOWN, 0x805, METHOD_BUFFERED ,FILE_ANY_ACCESS) -#define IOCTL_WSP_LISTEN CTL_CODE(FILE_DEVICE_UNKNOWN, 0x806, METHOD_BUFFERED ,FILE_ANY_ACCESS) -#define IOCTL_WSP_ACCEPT CTL_CODE(FILE_DEVICE_UNKNOWN, 0x807, METHOD_BUFFERED ,FILE_ANY_ACCESS) - +#define IOCTL_WSP_SOCKET CTL_CODE(FILE_DEVICE_UNKNOWN, 0x801, METHOD_BUFFERED ,FILE_ANY_ACCESS) +#define IOCTL_WSP_CONNECT CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED ,FILE_ANY_ACCESS) +#define IOCTL_WSP_SEND CTL_CODE(FILE_DEVICE_UNKNOWN, 0x803, METHOD_BUFFERED ,FILE_ANY_ACCESS) +#define IOCTL_WSP_RECV CTL_CODE(FILE_DEVICE_UNKNOWN, 0x804, METHOD_BUFFERED ,FILE_ANY_ACCESS) +#define IOCTL_WSP_BIND CTL_CODE(FILE_DEVICE_UNKNOWN, 0x805, METHOD_BUFFERED ,FILE_ANY_ACCESS) +#define IOCTL_WSP_LISTEN CTL_CODE(FILE_DEVICE_UNKNOWN, 0x806, METHOD_BUFFERED ,FILE_ANY_ACCESS) +#define IOCTL_WSP_ACCEPT CTL_CODE(FILE_DEVICE_UNKNOWN, 0x807, METHOD_BUFFERED ,FILE_ANY_ACCESS) +#define IOCTL_WSP_CLOSE_SOCKET CTL_CODE(FILE_DEVICE_UNKNOWN, 0x808, METHOD_BUFFERED ,FILE_ANY_ACCESS) // Data structures that are used for connect @@ -96,5 +96,14 @@ struct WspAcceptOut { USHORT Port; }; +struct WspSocketCloseIn { + VOID *pSocket; +}; + +struct WspSocketCloseOut { + int Errno; +}; + + #endif //_SDP_SHARED_H diff --git a/trunk/ulp/sdp/kernel/SdpArp.h b/trunk/ulp/sdp/kernel/SdpArp.h index f869d567..09b92a33 100644 --- a/trunk/ulp/sdp/kernel/SdpArp.h +++ b/trunk/ulp/sdp/kernel/SdpArp.h @@ -46,6 +46,17 @@ public: + 159; return STATUS_SUCCESS; } + + if (DestIp == 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 157) { + *SrcIp = 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 154; + return STATUS_SUCCESS; + } ASSERT(FALSE); *SrcIp = 0; @@ -92,6 +103,26 @@ public: *SrcCaGuid = CL_NTOH64(0x2c901093d8430); return STATUS_SUCCESS; } + + if (SourceAddr == 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 154) { + + *SrcPortGuid = CL_NTOH64(0x2c9000100d151);//????? swlab63 + *SrcCaGuid = CL_NTOH64(0x2c9000100d150); + return STATUS_SUCCESS; + } + + if (SourceAddr == 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 157) { + + *SrcPortGuid = CL_NTOH64(0x2c9000100d051);//????? swlab63 + *SrcCaGuid = CL_NTOH64(0x2c9000100d050); + return STATUS_SUCCESS; + } ASSERT(FALSE); *SrcPortGuid = 0; @@ -101,6 +132,7 @@ public: } + // Get the remote GID based on it's IP addresses NTSTATUS DestPortGidFromIP( IN ULONG DestAddr, OUT ib_net64_t *DestPortGuid) @@ -132,6 +164,16 @@ public: return STATUS_SUCCESS; } + + if (DestAddr == 11 * 256*256*256 + + 4 * 256*256 + + 8 * 256 + + + 157) { + *DestPortGuid = CL_NTOH64(0x2c9000100d051);//????? swlab57 + + return STATUS_SUCCESS; + } + ASSERT(FALSE); *DestPortGuid = 0; diff --git a/trunk/ulp/sdp/kernel/SdpBufferPool.cpp b/trunk/ulp/sdp/kernel/SdpBufferPool.cpp index b2e2ecb3..dbf8f3b5 100644 --- a/trunk/ulp/sdp/kernel/SdpBufferPool.cpp +++ b/trunk/ulp/sdp/kernel/SdpBufferPool.cpp @@ -8,7 +8,7 @@ NTSTATUS BufferDescriptor::AllocateBuffer(BufferDescriptor ** ppBufferDescriptor, int BufferSize, int Tag) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("\n")); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("\n")); NTSTATUS rc = STATUS_SUCCESS; BufferDescriptor *pBufferDescriptor = NULL; @@ -20,22 +20,21 @@ BufferDescriptor::AllocateBuffer(BufferDescriptor ** ppBufferDescriptor, int Buf sizeof BufferDescriptor, Tag ); + if (pBufferDescriptor == NULL) { SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("ExAllocatePoolWithTag failed \n")); rc = STATUS_NO_MEMORY; goto Cleanup; } - // Allocate the buffer itself - pBufferDescriptor->pBuffer = - ExAllocatePoolWithTag( - NonPagedPool , - BufferSize, - Tag - ); - + // Allocate the buffer itself (from continious memory) + PHYSICAL_ADDRESS mem; + mem.HighPart = 0xffffffff; + mem.LowPart = 0xffffffff; + pBufferDescriptor->pBuffer = MmAllocateContiguousMemory(BufferSize, mem); + if (pBufferDescriptor->pBuffer == NULL) { - SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("ExAllocatePoolWithTag failed \n")); + SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("ExAllocatePoolWithTag failed BufferSize = %d irql=%d\n",BufferSize, KeGetCurrentIrql())); rc = STATUS_NO_MEMORY; goto Cleanup; } @@ -47,7 +46,7 @@ Cleanup: if (!NT_SUCCESS(rc)) { if (pBufferDescriptor != NULL) { if (pBufferDescriptor->pBuffer != NULL) { - ExFreePoolWithTag(pBufferDescriptor->pBuffer, SEND_BUFFERS_ALLOCATION_TAG); + MmFreeContiguousMemory(pBufferDescriptor->pBuffer); } ExFreePoolWithTag(pBufferDescriptor, SEND_BUFFERS_ALLOCATION_TAG); pBufferDescriptor = NULL; @@ -62,7 +61,7 @@ Cleanup: VOID BufferDescriptor::DeAllocateBuffer(BufferDescriptor *pBufferDescriptor, int Tag) { - ExFreePoolWithTag(pBufferDescriptor->pBuffer, Tag); + MmFreeContiguousMemory(pBufferDescriptor->pBuffer); ExFreePoolWithTag(pBufferDescriptor, Tag); } @@ -130,7 +129,7 @@ BufferPool::GetBuffer( bool FirstBuffer ) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p FirstBuffer = %s\n",this, + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p FirstBuffer = %s\n",this, FirstBuffer ? "TRUE" : "FALSE")); AssertLocked(); @@ -187,12 +186,13 @@ queue NTSTATUS BufferPool::AddBufferToQueuedList(BufferDescriptor *pBufferDescriptor) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p pBufferDescriptor = 0x%x\n",this, + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p pBufferDescriptor = 0x%x\n",this, pBufferDescriptor)); AssertLocked(); NTSTATUS rc = STATUS_SUCCESS; - ASSERT(pBufferDescriptor->GetFlags() == 0); + ASSERT(pBufferDescriptor->GetFlags() == 0 || + pBufferDescriptor->GetFlags() == DISCONNECT_MESSAGE); m_QueuedPackets.InsertTailList(&pBufferDescriptor->BuffersList); rc = SendBuffersIfCan(); @@ -227,9 +227,10 @@ BufferPool::AllowOthersToGet() NTSTATUS BufferPool::ReturnBuffer(BufferDescriptor *pBufferDescriptor) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p buffer=0x%p\n",this, pBufferDescriptor)); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p buffer=0x%p\n",this, pBufferDescriptor)); AssertLocked(); bool CreditUpdate = false; + bool DissconnectMessage = false; NTSTATUS rc = STATUS_SUCCESS; #if DBG @@ -239,9 +240,11 @@ BufferPool::ReturnBuffer(BufferDescriptor *pBufferDescriptor) } #endif ASSERT( pBufferDescriptor->GetFlags() == CREDIT_UPDATE || + pBufferDescriptor->GetFlags() == DISCONNECT_MESSAGE || pBufferDescriptor->GetFlags() == 0); CreditUpdate = (pBufferDescriptor->GetFlags() == CREDIT_UPDATE); + DissconnectMessage = (pBufferDescriptor->GetFlags() == DISCONNECT_MESSAGE); if (CreditUpdate) { // This is a credit update packet, need to act accordingly ASSERT(m_CreditdBufferDescriptor == NULL); @@ -264,6 +267,10 @@ BufferPool::ReturnBuffer(BufferDescriptor *pBufferDescriptor) KeSetEvent( &m_WaitingClients, IO_NO_INCREMENT, FALSE ); m_ClientWaiting = false; } + if (DissconnectMessage) { + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("We have recieved a DissconnectMessage complition\n" )); + m_pSdpSocket->DisconectSentEvent(); + } } Cleanup: m_CurrentlySentBuffers--; @@ -281,7 +288,7 @@ Cleanup: NTSTATUS BufferPool::SendBuffersIfCan() { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); AssertLocked(); NTSTATUS rc = STATUS_SUCCESS; @@ -304,6 +311,30 @@ Cleanup: } + +/* This function is called when the user mode has called close socket + If a client is waiting on recieve we free him, he should get an + error on his callback. *** The caller probably has a bug - as one + can't race a closesocket *** +*/ + +VOID +BufferPool::CloseSocket() +{ + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p m_WaitingClients = %s\n", + m_ClientWaiting ? "true" : "false")); + AssertLocked(); + + if (m_ClientWaiting) { + KeSetEvent( &m_WaitingClients, IO_NO_INCREMENT, FALSE ); + m_ClientWaiting = false; + } + // The next time our client will try to get data, he will get + // the error +} + + + /* This function is being called from under the lock and is the last one to be called. It frees all resources @@ -339,7 +370,7 @@ BufferPool::ShutDown() NTSTATUS BufferPool::SendBuffer(BufferDescriptor *pBufferDescriptor) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); AssertLocked(); NTSTATUS rc = STATUS_SUCCESS; @@ -350,7 +381,6 @@ BufferPool::SendBuffer(BufferDescriptor *pBufferDescriptor) pHeader->seq_num = GetAndIncreaseSendSeq(); pHeader->seq_ack = m_pSdpSocket->m_RecvBufferPool.GetRecvSeq(); m_AdvtSeq = pHeader->seq_ack;// Currently only for debug - pHeader->mid = SDP_MID_DATA; pHeader->flags = SDP_MSG_FLAG_NON_FLAG; /* * endian swap @@ -392,11 +422,10 @@ Cleanup: return rc; } - NTSTATUS BufferPool::PostCredits() { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); AssertLocked(); NTSTATUS rc = STATUS_SUCCESS; @@ -415,7 +444,7 @@ BufferPool::PostCredits() rc = BufferDescriptor::AllocateBuffer( &m_CreditdBufferDescriptor, - sizeof msg_hdr_bsdh, + sizeof msg_hdr_bsdh, SEND_BUFFERS_ALLOCATION_TAG ); if (!NT_SUCCESS(rc)) { @@ -429,6 +458,7 @@ BufferPool::PostCredits() ASSERT(m_CreditdBufferDescriptor->DataSize == 0); + m_CreditdBufferDescriptor->SetMid(SDP_MID_DATA); rc = SendBuffer(m_CreditdBufferDescriptor); if (!NT_SUCCESS(rc)) { SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("SendBuffer failed rc = 0x%x\n", rc )); @@ -443,6 +473,41 @@ Cleanup: } +NTSTATUS BufferPool::PostDisConn() +{ + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); + AssertLocked(); + NTSTATUS rc = STATUS_SUCCESS; + BufferDescriptor *pBufferDescriptor = NULL; + + rc = BufferDescriptor::AllocateBuffer( + &pBufferDescriptor, + sizeof msg_hdr_bsdh, + SEND_BUFFERS_ALLOCATION_TAG + ); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("AllocateBuffer failed rc = 0x%x\n", rc )); + ASSERT(m_CreditdBufferDescriptor == NULL); + goto Cleanup; + } + + pBufferDescriptor->SetFlags(DISCONNECT_MESSAGE); + + ASSERT(pBufferDescriptor->DataSize == 0); + + pBufferDescriptor->SetMid(SDP_MID_DISCONNECT); + rc = AddBufferToQueuedList(pBufferDescriptor); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("SendBuffer failed rc = 0x%x\n", rc )); + goto Cleanup; + } + +Cleanup: + return rc; + +} + + VOID BufferPool::AssertLocked() { #if DBG diff --git a/trunk/ulp/sdp/kernel/SdpBufferPool.h b/trunk/ulp/sdp/kernel/SdpBufferPool.h index 05940a0d..deccc6a5 100644 --- a/trunk/ulp/sdp/kernel/SdpBufferPool.h +++ b/trunk/ulp/sdp/kernel/SdpBufferPool.h @@ -69,6 +69,7 @@ typedef void (* SendErrorCB )(NTSTATUS Error, VOID *Context); // The flags that are being used to give more information about the BufferDescriptors const uint8_t CREDIT_UPDATE = 1; +const uint8_t DISCONNECT_MESSAGE = 2; // Each buffer starts with msg_hdr_bsdh and is followed by the actual data @@ -114,6 +115,11 @@ public: VOID SetFlags(uint8_t flags) { Flags = flags; } uint8_t GetFlags() { return Flags; } + VOID SetMid(uint8_t Mid) { + msg_hdr_bsdh *pHeader = (msg_hdr_bsdh *) pBuffer; + pHeader->mid = Mid; + } + // Each buffer starts with bsdh_hdr structure VOID *pBuffer; // A pointer to the actual place that we put the data uint32_t BufferSize; // The total size of the buffer (size that we have allocated) @@ -161,6 +167,8 @@ public: NTSTATUS SendBuffersIfCan(); + VOID CloseSocket(); + VOID ShutDown(); uint32_t GetSendSeq() {return m_SendSeq;} @@ -176,6 +184,8 @@ public: } NTSTATUS PostCredits(); + + NTSTATUS PostDisConn(); private: diff --git a/trunk/ulp/sdp/kernel/SdpConnectionList.cpp b/trunk/ulp/sdp/kernel/SdpConnectionList.cpp index 7920fefc..a56c714a 100644 --- a/trunk/ulp/sdp/kernel/SdpConnectionList.cpp +++ b/trunk/ulp/sdp/kernel/SdpConnectionList.cpp @@ -11,6 +11,29 @@ ConnectionList::Init(SdpSocket *pSdpSocket) KeInitializeEvent(&m_WaitForConnection, NotificationEvent , FALSE ); } + +/* This function is called when the user mode has called close socket + If a client is waiting on recieve we free him, he should get an + error on his callback. *** The caller probably has a bug - as one + can't race a closesocket *** +*/ + +VOID +ConnectionList::CloseSocket() +{ + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p m_WaitingClients = %s\n", + m_ClientWaiting ? "true" : "false")); + AssertLocked(); + + if (m_ClientWaiting) { + KeSetEvent( &m_WaitForConnection, IO_NO_INCREMENT, FALSE ); + m_ClientWaiting = false; + } + // The next time our client will try to get data, he will get + // the error +} + + VOID ConnectionList::Shutdown() { //?????AssertLocked(); diff --git a/trunk/ulp/sdp/kernel/SdpConnectionList.h b/trunk/ulp/sdp/kernel/SdpConnectionList.h index cb95896f..d6b87592 100644 --- a/trunk/ulp/sdp/kernel/SdpConnectionList.h +++ b/trunk/ulp/sdp/kernel/SdpConnectionList.h @@ -10,6 +10,9 @@ class ConnectionList { public: VOID Init(SdpSocket *pSdpSocket); + + VOID CloseSocket(); + VOID Shutdown(); bool IsFull(); diff --git a/trunk/ulp/sdp/kernel/SdpDriver.cpp b/trunk/ulp/sdp/kernel/SdpDriver.cpp index 29a12710..40e87840 100644 --- a/trunk/ulp/sdp/kernel/SdpDriver.cpp +++ b/trunk/ulp/sdp/kernel/SdpDriver.cpp @@ -135,8 +135,6 @@ NTSTATUS SdpDriver::Dispatch( int Method; SdpUserFile *pSdpUserFile = NULL; - - pIrpSp = IoGetCurrentIrpStackLocation(pIrp); switch (pIrpSp->MajorFunction) { @@ -192,7 +190,7 @@ NTSTATUS SdpDriver::Dispatch( break; } case IRP_MJ_DEVICE_CONTROL: { - SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("IRP_MJ_DEVICE_CONTROL pIrpSp->FileObject = 0x%x\n", + SDP_PRINT(SDP_DEBUG, SDP_DRIVER, ("IRP_MJ_DEVICE_CONTROL pIrpSp->FileObject = 0x%x\n", pIrpSp->FileObject )); // IOCTLs are allowed only for user mode processes @@ -344,7 +342,7 @@ NTSTATUS SdpDriver::DispatchDeviceIoControl( switch (IoControlCode) { case IOCTL_WSP_SOCKET : { - SDP_PRINT(SDP_DEBUG, SDP_DRIVER, ("IOCTL_WSP_SOCKET recieved\n" )); + SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("IOCTL_WSP_SOCKET recieved\n" )); VERIFY_BUFFERS(InputBufferLength, OutputBufferLength, WspSocketIn, WspSocketOut); OutputDataSize = sizeof (WspSocketOut); @@ -374,7 +372,7 @@ NTSTATUS SdpDriver::DispatchDeviceIoControl( case IOCTL_WSP_CONNECT : { - SDP_PRINT(SDP_DEBUG, SDP_DRIVER, ("IOCTL_WSP_CONNECT recieved\n" )); + SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("IOCTL_WSP_CONNECT recieved\n" )); VERIFY_BUFFERS(InputBufferLength, OutputBufferLength, WspConnectIn, WspConnectOut); OutputDataSize = sizeof (WspConnectOut); @@ -449,7 +447,7 @@ NTSTATUS SdpDriver::DispatchDeviceIoControl( case IOCTL_WSP_BIND: { - SDP_PRINT(SDP_DEBUG, SDP_DRIVER, ("IOCTL_WSP_BIND recieved\n" )); + SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("IOCTL_WSP_BIND recieved\n" )); VERIFY_BUFFERS(InputBufferLength, OutputBufferLength, WspBindIn, WspBindOut); OutputDataSize = sizeof (WspBindOut); @@ -474,7 +472,7 @@ NTSTATUS SdpDriver::DispatchDeviceIoControl( case IOCTL_WSP_LISTEN: { - SDP_PRINT(SDP_DEBUG, SDP_DRIVER, ("IOCTL_WSP_LISTEN recieved\n" )); + SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("IOCTL_WSP_LISTEN recieved\n" )); VERIFY_BUFFERS(InputBufferLength, OutputBufferLength, WspListenIn, WspListenOut); OutputDataSize = sizeof (WspListenOut); @@ -499,7 +497,7 @@ NTSTATUS SdpDriver::DispatchDeviceIoControl( case IOCTL_WSP_ACCEPT: { - SDP_PRINT(SDP_DEBUG, SDP_DRIVER, ("IOCTL_WSP_ACCEPT recieved\n" )); + SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("IOCTL_WSP_ACCEPT recieved\n" )); VERIFY_BUFFERS(InputBufferLength, OutputBufferLength, WspAcceptIn, WspAcceptOut); OutputDataSize = sizeof (WspAcceptOut); @@ -530,7 +528,35 @@ NTSTATUS SdpDriver::DispatchDeviceIoControl( } } break; - + + case IOCTL_WSP_CLOSE_SOCKET : + { + SDP_PRINT(SDP_TRACE, SDP_DRIVER, ("IOCTL_WSP_CLOSE_SOCKET recieved\n" )); + VERIFY_BUFFERS(InputBufferLength, OutputBufferLength, WspSocketCloseIn, WspSocketCloseOut); + OutputDataSize = sizeof (WspSocketCloseOut); + + // get the socket based on the users pointer + WspSocketCloseIn wspSocketCloseIn = *(WspSocketCloseIn *) pInputBuffer; + WspSocketCloseOut *pWspSocketCloseOut = (WspSocketCloseOut *) pOutputBuffer; + pSdpUserFile = (SdpUserFile *)pIrpSp->FileObject->FsContext; + pSdpSocket = pSdpUserFile->SocketByPointer(wspSocketCloseIn.pSocket); + if (pSdpSocket == NULL) { + SDP_PRINT(SDP_DEBUG, SDP_DRIVER, ("IOCTL_WSP_CLOSE_SOCKET socket %x not found\n",wspSocketCloseIn.pSocket)); + // This is a well defined winsock error + pWspSocketCloseOut->Errno = WSAENOTSOCK; + goto Cleanup; + } + rc = pSdpSocket->WSPCloseSocket(&wspSocketCloseIn, pWspSocketCloseOut); + // After closing a socket we "unlink" the kernel object, and it won't + // be accessable for the user. (currently succesfull or not) + // BUGBUG: Change this behavior while the linger don't linger staff is fixed + pSdpUserFile->RemoveSocket(pSdpSocket); // Must succed + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_DRIVER, ("pSdpSocket->WSPCloseSocket failed rc = 0x%x\n", rc )); + goto Cleanup; + } + } + break; default: // This is an unrecgnized IOCTL diff --git a/trunk/ulp/sdp/kernel/SdpGenUtils.cpp b/trunk/ulp/sdp/kernel/SdpGenUtils.cpp index cad9a834..34ccce21 100644 --- a/trunk/ulp/sdp/kernel/SdpGenUtils.cpp +++ b/trunk/ulp/sdp/kernel/SdpGenUtils.cpp @@ -73,8 +73,9 @@ NTSTATUS } } if (i == 20) { - SDP_PRINT(SDP_ERR, SDP_SOCKET, ("!!!! KeWaitForSingleObject was Exhausted STATUS_USER_APC\n" )); - + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("!!!! KeWaitForSingleObject was Exhausted STATUS_USER_APC\n" )); + // This is probably fine if we are runnign for a user thread + ASSERT((WaitReason == UserRequest) && (WaitMode == UserMode)); } return rc; } @@ -141,6 +142,35 @@ int abs(int i){ return -i; } +LARGE_INTEGER TimeFromLong(ULONG HandredNanos) +{ + LARGE_INTEGER Timeout; + Timeout.HighPart = 0xffffffff; + Timeout.LowPart = 0xffffffff ^ HandredNanos; + return Timeout; +} + +NTSTATUS Sleep(ULONG HandredNanos) +{ + KEVENT Event; + NTSTATUS rc = STATUS_SUCCESS; + + KeInitializeEvent(&Event, NotificationEvent , FALSE ); + + LARGE_INTEGER Timeout = TimeFromLong(HandredNanos); + + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("Before Sleep\n")); + rc = MyKeWaitForSingleObject( + &Event, + UserRequest, + UserMode, + FALSE, + &Timeout); + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("After Sleep rc = 0x%x\n", rc)); + return rc; + +} + /* Convert an IBAL error to a Winsock error. */ int IbalToWsaError(const ib_api_status_t ib_status ) { diff --git a/trunk/ulp/sdp/kernel/SdpGenUtils.h b/trunk/ulp/sdp/kernel/SdpGenUtils.h index 98f2256d..ebd95ff8 100644 --- a/trunk/ulp/sdp/kernel/SdpGenUtils.h +++ b/trunk/ulp/sdp/kernel/SdpGenUtils.h @@ -72,6 +72,11 @@ VOID UpdateRc(NTSTATUS *rc, NTSTATUS rc1); int abs(int i); +LARGE_INTEGER TimeFromLong(ULONG HandredNanos); + +NTSTATUS Sleep(ULONG HandredNanos); + + /* Convert an IBAL error to a Winsock error. */ int IbalToWsaError(const ib_api_status_t ib_status ); diff --git a/trunk/ulp/sdp/kernel/SdpLock.h b/trunk/ulp/sdp/kernel/SdpLock.h index b833d4db..1eeb024e 100644 --- a/trunk/ulp/sdp/kernel/SdpLock.h +++ b/trunk/ulp/sdp/kernel/SdpLock.h @@ -221,10 +221,8 @@ Cleanup: ASSERT(m_flags & ERROR_SIGNALLED); } KeAcquireSpinLock(&m_SpinLock, &OldIrql); - int xxx = m_flags; if ((m_flags & DPC_FLAGS) == 0) { // No flags to handle from the DPC layer - ASSERT(m_flags == 0); break; } } diff --git a/trunk/ulp/sdp/kernel/SdpRecvPool.cpp b/trunk/ulp/sdp/kernel/SdpRecvPool.cpp index d25f15e7..bd3372cd 100644 --- a/trunk/ulp/sdp/kernel/SdpRecvPool.cpp +++ b/trunk/ulp/sdp/kernel/SdpRecvPool.cpp @@ -9,6 +9,7 @@ RecvPool::RecvPool() m_CurrentlyPostedRecievedBuffers = 0; m_CurrentlyAllocated = 0; m_ClientWaiting = false; + m_DisConnRecieved = false; m_LocaleAdvertisedBuffers = 0; } @@ -23,7 +24,7 @@ RecvPool::Init( SdpSocket *pSdpSocket ) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n", m_pSdpSocket)); m_MaxBuffers = MaxBuffers; m_MaxConcurrentRecieves = MaxConcurrentRecieves; m_MaxMessageSize = MaxMessageSize; @@ -35,6 +36,23 @@ RecvPool::Init( ASSERT(lkey != NULL); m_lkey = lkey; m_pSdpSocket = pSdpSocket; + + // Allocate all the buffers as continues memory (better be done + // at passive level) + BufferDescriptor *pBufferDescriptor = NULL; + NTSTATUS rc = STATUS_SUCCESS; + + for (int i=0;i < MAX_RECV_PACKETS; i++) { + rc = BufferDescriptor::AllocateBuffer(&pBufferDescriptor, m_MaxMessageSize, SEND_BUFFERS_ALLOCATION_TAG); + ASSERT(NT_SUCCESS(rc)); + m_CurrentlyAllocated++; + + pBufferDescriptor->Reset(); + m_FreePackets.InsertTailList(&pBufferDescriptor->BuffersList); + } + m_FirstPostingOfBuffers = 0; + + return STATUS_SUCCESS; } @@ -45,8 +63,8 @@ RecvPool::Init( NTSTATUS RecvPool::RecievedBuffer(BufferDescriptor *pBufferDescriptor, bool error) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p pBufferDescriptor = 0x%x error = %s\n" - ,this, pBufferDescriptor, error ? "true" : "false")); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p pBufferDescriptor = 0x%p error = %s\n" + ,m_pSdpSocket, pBufferDescriptor, error ? "true" : "false")); AssertLocked(); NTSTATUS rc = STATUS_SUCCESS; @@ -84,8 +102,18 @@ RecvPool::RecievedBuffer(BufferDescriptor *pBufferDescriptor, bool error) (int)pHeader->seq_ack)); m_pSdpSocket->m_SendBufferPool.SetRemoteRecvBuf(rRecvBuf); - // ???? Handle state changes here ???? - if (pHeader->mid != 0xFF) { + // m_DisConnRecieved is the last message that should be recieved + ASSERT(m_DisConnRecieved == false); // BUGBUG: do a real check here + + // ???? Handle more state changes here ???? + if (pHeader->mid != SDP_MID_DATA) { + if (pHeader->mid == SDP_MID_DISCONNECT) { + m_DisConnRecieved = true; + if (m_ClientWaiting) { + KeSetEvent( &m_WaitingClients, IO_NO_INCREMENT, FALSE ); + m_ClientWaiting = false; + } + } // This is a message without data, we currently ignore it m_FreePackets.InsertTailList(&pBufferDescriptor->BuffersList); } else { @@ -163,7 +191,7 @@ RecvPool::GetData( bool *pNoMoreData ) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p FirstBuffer = %s\n",this, + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p FirstBuffer = %s\n", m_pSdpSocket, FirstBuffer ? "TRUE" : "FALSE")); AssertLocked(); bool BufferFreed = false; @@ -221,7 +249,10 @@ RecvPool::GetData( } else { *pNoMoreData = true; // If data was already supplied, then we shouldn't wait any more - if ((*Copied > 0) || (FirstBuffer == false)) { + if ((*Copied > 0) || + (FirstBuffer == false) || + m_DisConnRecieved) { + ASSERT(*ppEvent == NULL); break; } // No buffers available, we have to wait @@ -245,7 +276,7 @@ Cleanup: VOID RecvPool::AllowOthersToGet() { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p \n", m_pSdpSocket)); ASSERT(m_ClientBeingServed == true); m_ClientBeingServed = false; @@ -256,8 +287,8 @@ RecvPool::AllowOthersToGet() NTSTATUS RecvPool::ReceiveIfCan() { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p m_CurrentlyPostedRecievedBuffers = %d m_LocaleAdvertisedBuffers = %d\n", - this, m_CurrentlyPostedRecievedBuffers, m_LocaleAdvertisedBuffers)); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p m_CurrentlyPostedRecievedBuffers = %d m_LocaleAdvertisedBuffers = %d\n", + m_pSdpSocket, m_CurrentlyPostedRecievedBuffers, m_LocaleAdvertisedBuffers)); AssertLocked(); BufferDescriptor *pBufferDescriptor = NULL; NTSTATUS rc = STATUS_SUCCESS; @@ -266,6 +297,13 @@ RecvPool::ReceiveIfCan() while (m_CurrentlyPostedRecievedBuffers < m_MaxConcurrentRecieves) { // do we have a free packet ? if (m_FreePackets.Size() > 0) { + //??????????? lET'S Sleep here for some time (FIND THE RACE) ?????????? + { + if (m_FirstPostingOfBuffers ==0 ) { + m_FirstPostingOfBuffers++; + Sleep(0xfffff); + } + } // we can take a packet from the list LIST_ENTRY *item = m_FreePackets.RemoveHeadList(); pBufferDescriptor = CONTAINING_RECORD(item, BufferDescriptor , BuffersList); @@ -313,8 +351,9 @@ Cleanup: VOID RecvPool::ShutDown() { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); - //???? AssertLocked(); + // Lock is not taken here, but we should be gurantied that no + // one can be racing us in here + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n", m_pSdpSocket)); BufferDescriptor *pBufferDescriptor = NULL; LIST_ENTRY *item = NULL; @@ -331,10 +370,34 @@ RecvPool::ShutDown() } } +/* This function is called when the user mode has called close socket + If a client is waiting on recieve we free him, he should get an + error on his callback. *** The caller probably has a bug - as one + can't race a closesocket *** +*/ + +VOID +RecvPool::CloseSocket() +{ + SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p m_WaitingClients = %s waiting buffer = %d\n", + this, + m_ClientWaiting ? "true" : "false", + m_FullPackets.Size()) + ); + AssertLocked(); + + if (m_ClientWaiting) { + KeSetEvent( &m_WaitingClients, IO_NO_INCREMENT, FALSE ); + m_ClientWaiting = false; + } + // The next time our client will try to get data, he will get + // the error +} + NTSTATUS RecvPool::PostReceiveBuffer(BufferDescriptor *pBufferDescriptor) { - SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this)); + SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL, ("this = 0x%p \n", m_pSdpSocket)); AssertLocked(); NTSTATUS rc = STATUS_SUCCESS; diff --git a/trunk/ulp/sdp/kernel/SdpRecvPool.h b/trunk/ulp/sdp/kernel/SdpRecvPool.h index 715b03ae..625b94c6 100644 --- a/trunk/ulp/sdp/kernel/SdpRecvPool.h +++ b/trunk/ulp/sdp/kernel/SdpRecvPool.h @@ -36,6 +36,8 @@ public: uint32_t GetRecvSeq() { return m_RecvSeq;} + VOID CloseSocket(); + VOID ShutDown(); uint16_t GetCurrentlyPostedRecievedBuffers() { @@ -85,6 +87,11 @@ private: SdpSocket *m_pSdpSocket; + // This signals that the remote side will not be sending any data any more + bool m_DisConnRecieved; + + int m_FirstPostingOfBuffers;//????????????????? USED TO HANDLE SOME RACE + VOID AssertLocked(); }; diff --git a/trunk/ulp/sdp/kernel/SdpSocket.cpp b/trunk/ulp/sdp/kernel/SdpSocket.cpp index d6d7e9ec..6bdccb31 100644 --- a/trunk/ulp/sdp/kernel/SdpSocket.cpp +++ b/trunk/ulp/sdp/kernel/SdpSocket.cpp @@ -48,7 +48,9 @@ static void AL_API cm_dreq_callback(IN ib_cm_dreq_rec_t *p_cm_dreq_rec ) { SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("dispatch level = %d\n", KeGetCurrentIrql())); - ASSERT(FALSE); + SdpSocket *pSocket = (SdpSocket *) p_cm_dreq_rec->qp_context; + pSocket->CmDreqCallback(p_cm_dreq_rec); + } static void AL_API @@ -85,6 +87,12 @@ listen_err_callback( ASSERT( 0 ); } +void CloseSocketThread(void *pContext) +{ + SdpSocket *pSocket = (SdpSocket *) pContext; + pSocket->CloseSocketThread(); + +} static void AL_API cm_req_callback( IN ib_cm_req_rec_t *p_cm_req_rec ) @@ -115,6 +123,11 @@ SdpSocket::SdpSocket() m_state = SS_IDLE; m_pListeningSocket = NULL; + + m_CloseSocketCalled = false; + m_ShutdownCalled = false; + m_DisconnectConnectionRecieved = false; + } VOID SdpSocket::AssertLocked() @@ -137,6 +150,7 @@ NTSTATUS SdpSocket::Init( pSocketOutParam->Errno = 0;// No error pSocketOutParam->pSocket = this; // give the user a handle to the socket KeInitializeEvent(&m_ShutdownCompleteEvent, NotificationEvent , FALSE ); + KeInitializeEvent(&m_DisconectSentEvent, NotificationEvent , FALSE ); m_ConnectionList.Init(this); @@ -148,7 +162,7 @@ NTSTATUS SdpSocket::WSPSend( WspSendOut *pWspSendOut ) { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p \n",this)); + SDP_PRINT(SDP_DEBUG, SDP_SOCKET, ("this = 0x%p \n",this)); NTSTATUS rc = STATUS_SUCCESS; NTSTATUS rc1; // used only to check that there are no more errors on the @@ -175,8 +189,18 @@ NTSTATUS SdpSocket::WSPSend( Locked = true; ASSERT(pBuffersEvent == NULL); - //??? Verify connected state (or whatever)????????????? - + if ((m_state != SS_CONNECTED)) { + // We can not send now. + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("Can't send now, m_state = %s\n", + SS2String(m_state) + )); + rc = STATUS_SHUTDOWN_IN_PROGRESS; + pWspSendOut->Errno = WSAENOTCONN; + + m_Lock.Unlock(); // Error ignored as this is already an error pass + Locked = false; + goto Cleanup; + } rc = m_SendBufferPool.GetBuffer(&pBufferDescriptor, &pBuffersEvent, First); if (!NT_SUCCESS(rc)) { @@ -207,7 +231,7 @@ NTSTATUS SdpSocket::WSPSend( if (( rc == STATUS_ALERTED ) ||( rc == STATUS_USER_APC )) { // BUGBUG: Think what to do here, we should be able to stop the // connect, and quit (probably shutdown should be enough) - SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted = 0x%x\n", rc )); + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted rc = 0x%x\n", rc )); rc = STATUS_UNEXPECTED_IO_ERROR; //pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error Shutdown(); @@ -232,7 +256,8 @@ NTSTATUS SdpSocket::WSPSend( } Coppied += CopySize; - // return the data to the buffer + // send the data to the buffer + pBufferDescriptor->SetMid(SDP_MID_DATA); rc = m_SendBufferPool.AddBufferToQueuedList(pBufferDescriptor); if (!NT_SUCCESS(rc)) { SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_SendBufferPool.AddBufferToQueuedList failed rc = 0x%x\n", rc )); @@ -271,7 +296,7 @@ SdpSocket::WSPRecv( WspRecvOut *pWspRecvOut ) { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p BufferSize = %d\n",this, pWspRecvIn->BufferSize)); + SDP_PRINT(SDP_DEBUG, SDP_SOCKET, ("this = 0x%p BufferSize = %d\n",this, pWspRecvIn->BufferSize)); NTSTATUS rc = STATUS_SUCCESS; bool First = true; @@ -294,7 +319,18 @@ SdpSocket::WSPRecv( Locked = true; ASSERT(pBuffersEvent == NULL); - //??? Verify connected state (or whatever)????????????? + if ((m_state != SS_CONNECTED && m_state!= SS_CONNECTED_DREP_SENT ) ) { + // We can not recv now. + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("Can't recv now, m_state = %s\n", + SS2String(m_state) + )); + rc = STATUS_SHUTDOWN_IN_PROGRESS; + pWspRecvOut->Errno = WSAENOTCONN; + + m_Lock.Unlock(); // Error ignored as this is already an error pass + Locked = false; + goto Cleanup; + } rc = m_RecvBufferPool.GetData( pWspRecvIn->pData + Coppied, @@ -312,7 +348,7 @@ SdpSocket::WSPRecv( } Coppied += ThisCopy; - if (NoMoreData && (Coppied > 0)) { + if (NoMoreData && pBuffersEvent == NULL) { // this means that there is nothing to copy, and we should return ASSERT(pBuffersEvent == NULL); break; @@ -341,7 +377,7 @@ SdpSocket::WSPRecv( if (( rc == STATUS_ALERTED ) ||( rc == STATUS_USER_APC )) { // BUGBUG: Think what to do here, we should be able to stop the // connect, and quit (probably shutdown should be enough) - SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted = 0x%x\n", rc )); + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted rc = 0x%x\n", rc )); rc = STATUS_UNEXPECTED_IO_ERROR; //pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error Shutdown(); @@ -373,7 +409,7 @@ Cleanup: } // Currently in any case, the flags are not being used: pWspRecvOut->dwFlags = 0; - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p returning %d bytes \n",this, pWspRecvOut->NumberOfBytesRecieved)); + SDP_PRINT(SDP_DEBUG, SDP_SOCKET, ("this = 0x%p returning %d bytes \n",this, pWspRecvOut->NumberOfBytesRecieved)); return rc; } @@ -404,12 +440,15 @@ NTSTATUS SdpSocket::WSPConnect( } // check socket state - // BUGBUG: Do a better work here (it might be localy bounded ?????) m_Lock.Lock();//??? retval - if (m_state != SS_IDLE) { + if ((m_state != SS_IDLE)) { // We can not connect in this state - SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Invalid Socket state %s\n", SS2String(m_state))); - pWspConnectOut->Errno = WSAEINVAL; + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("Can't send now, m_state = %s\n", + SS2String(m_state) + )); + rc = STATUS_SHUTDOWN_IN_PROGRESS; + pWspConnectOut->Errno = WSAENOTCONN; + m_Lock.Unlock(); //?????retval goto Cleanup; @@ -525,7 +564,7 @@ NTSTATUS SdpSocket::WSPConnect( if (( rc == STATUS_ALERTED ) ||( rc == STATUS_USER_APC )) { // BUGBUG: Think what to do here, we should be able to stop the // connect, and quit (probably shutdown should be enough) - SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted = 0x%x\n", rc )); + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("MyKeWaitForSingleObject was alerted rc = 0x%x\n", rc )); rc = STATUS_UNEXPECTED_IO_ERROR; pWspConnectOut->Errno = WSAENETUNREACH; // BUGBUG: verify this error Shutdown(); @@ -877,10 +916,198 @@ Cleanup: // referance on pNewSocket is not currently released, since we are // returning them to the next level return rc; +} + +/* + CloseSocket is probably one of the most complicated winsock APIs. + Our current implmentation will be the default implmentation always. + This means that the call is always non-blocking, and it is always gracefully. + As a result, the remote side should return 0 on it's last read, and we should + send a DisConn mesage. + + Well, gracefull or not, the other side is not gourantied to call recieve and + we can't wait for ever. As a result a timer will be created and after 40 + seconds, we close abortively. + + If a different thread is waiting on some event (read, send or accept), we will + stop the blocking operation, and complete that wait with error. + + A special case to handle, is a socket that was created and after a while + closed. In this case, there is nothing really to do, and we will use the + same thread to close the socket. + + In the case of an error, an abortive close will be done (using the CM). + + There will be the following flag, telling the state of the socket. + + 1) CloseSocket called. + + So basicly, on close, we will do the following: + 1) Stop all blocking operations (send, recieve, accept). (ofcourse, new ones + are not allowed anymore) + 2) Create the DisConn packet and send it. (if needed) + 3) Start the timer, and wait for all the sends to end. + 4) When ever a send is complited, we will check if this a shutdown in progress + and we can close the socket. + + The "real" work of closing the socket will be done by the Shutdown function, + that will be called either by the CloseSocket when the logic is finished. + +*/ + +NTSTATUS +SdpSocket::WSPCloseSocket( + WspSocketCloseIn *pWspSocketCloseIn, + WspSocketCloseOut *pWspSocketCloseOut + ) +{ + NTSTATUS rc = STATUS_SUCCESS; + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p\n",this)); + OBJECT_ATTRIBUTES attr; + + if (!m_Lock.Lock()) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Failed to lock this = 0x%p \n",this)); + rc = STATUS_SHUTDOWN_IN_PROGRESS; + goto Cleanup; + } + + /* Verify the state of the socket */ + if(m_state == SS_IDLE) { + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("socket is in idle state this = 0x%p \n", + this)); + m_Lock.Unlock(); // Error ignored as this is already a Shutdown pass + Shutdown(); + pWspSocketCloseOut->Errno = 0; + goto Cleanup; + } + + // This will force that no more calls will be allowed + ASSERT(m_CloseSocketCalled == FALSE); // If this is not the case + // We shouldn't be able to take the lock + m_CloseSocketCalled = true; + + // notify to all "subclients" to free all waiting clients + m_RecvBufferPool.CloseSocket(); + m_SendBufferPool.CloseSocket(); + m_ConnectionList.CloseSocket(); + + if (m_state == SS_CONNECTED) { + // Need to send the DisConn message to the remote side and wait + rc = m_SendBufferPool.PostDisConn(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_SendBufferPool.PostDisConn failed rc = 0x%x\n", rc )); + m_Lock.Unlock(); // Error ignored as this is already an error pass + goto Cleanup; + } + } + + // We will now create a thread that will be resposible for the + // destruction of this socket + AddRef(); + + + /* Create a new thread, storing both the handle and thread id. */ + InitializeObjectAttributes( &attr, NULL, OBJ_KERNEL_HANDLE, NULL, NULL ); + + HANDLE ThreadHandle; + rc = PsCreateSystemThread( + &ThreadHandle, + THREAD_ALL_ACCESS, + &attr, + NULL, + NULL, + ::CloseSocketThread, + this + ); + + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("PsCreateSystemThread failed rc = 0x%x\n", rc )); + m_Lock.Unlock(); // Error ignored as this is already an error pass + // The thread wasn't created so we should remove the refferance + Release(); + goto Cleanup; + } + + // BUGBUG: Replace this with a mechanism that will allow + // to close the thered when the driver goes down + rc = ZwClose(ThreadHandle); + ASSERT(NT_SUCCESS(rc)); // Should succeed + + rc = m_Lock.Unlock(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_Lock.Unlock() failed rc = 0x%x\n", rc )); + goto Cleanup; + } + +Cleanup: + if (NT_SUCCESS(rc) ) { + pWspSocketCloseOut->Errno = 0; + } else { + // Make sure that we have the error setted + ASSERT(pWspSocketCloseOut->Errno != 0); // BUGBUG: Need to make sure that this + // is indeed the case. (also check if error was already signaled) + // In the current model, we call shutdown in any case (to kill the socket) + Shutdown(); + } + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p returning Errno = %d\n", + this , pWspSocketCloseOut->Errno)); + + + return rc; +} + +/* + This function is supposed to wait for the send to compleate + and then to kill the socket. + +*/ +VOID SdpSocket::DisconectSentEvent() +{ + KeSetEvent( &m_DisconectSentEvent, IO_NO_INCREMENT, FALSE ); } +/* + This is the "CloseSocket" thread call back. + This thread waits WAIT_TO_SOCKET_SHUTDOWN seconds for tht send to complete + and then it kills the socket abortively. +*/ +VOID SdpSocket::CloseSocketThread() +{ + NTSTATUS rc = STATUS_SUCCESS; + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p\n",this)); + + // wait for the last thread to end. + LARGE_INTEGER WaitTime; + + WaitTime = TimeFromLong(WAIT_TO_SOCKET_SHUTDOWN * 10 * 1000000); + rc = MyKeWaitForSingleObject( + &m_DisconectSentEvent, + Executive, + KernelMode, + FALSE, + &WaitTime + ); + + if (rc == STATUS_TIMEOUT) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Wait failed with time out\n")); + } + + ASSERT(NT_SUCCESS(rc)); + ASSERT(rc == STATUS_SUCCESS || rc == STATUS_TIMEOUT);// || rc == STATUS_USER_APC); //???????? what to do + + Shutdown(); + + // Everything done - the threads releases the last referance + // and kills itself + Release(); + PsTerminateSystemThread(STATUS_SUCCESS); + + // Do I get here ? + ASSERT(FALSE); + +} NTSTATUS SdpSocket::CmSendRTU() { @@ -946,9 +1173,6 @@ NTSTATUS SdpSocket::CmSendRTU() (u16)conn->send_size) - SDP_MSG_HDR_SIZE; #endif - /* - * Pop the hello message that was sent ????? I don't think that we need this - */ ib_cm_rtu_t cm_rtu; @@ -1012,7 +1236,7 @@ Cleanup: VOID SdpSocket::CmRepCallback(IN ib_cm_rep_rec_t *p_cm_rep_rec) { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%x\n", this)); + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p\n", this)); ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); if (m_state != SS_CONNECTING_REQ_SENT) { @@ -1034,7 +1258,7 @@ VOID SdpSocket::CmRepCallback(IN ib_cm_rep_rec_t *p_cm_rep_rec) VOID SdpSocket::CmReqCallback(IN ib_cm_req_rec_t *p_cm_req_rec) { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%x\n", this)); + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p\n", this)); NTSTATUS rc = STATUS_SUCCESS; ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); net64_t SrcCaGuid; @@ -1262,7 +1486,6 @@ SdpSocket::CmReqCallback(IN ib_cm_req_rec_t *p_cm_req_rec) if( ib_status != IB_SUCCESS ) { SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_rearm_cq failed ib_status = 0x%d\n", ib_status )); rc = IB2Status(ib_status); - pNewSocket->m_Lock.Unlock(); // Error is ignored, since this is already an error path goto ErrorLocked; } @@ -1302,7 +1525,7 @@ ErrorLocked: VOID SdpSocket::CmRtuCallback(IN ib_cm_rtu_rec_t *p_cm_rtu_rec) { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%x\n", this)); + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p\n", this)); NTSTATUS rc = STATUS_SUCCESS; ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); @@ -1389,6 +1612,74 @@ ErrorLocked: } +VOID +SdpSocket::CmDreqCallback(IN ib_cm_dreq_rec_t *p_cm_dreq_rec) +{ + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p\n", this)); + ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); + NTSTATUS rc = STATUS_SUCCESS; + ib_cm_drep_t cm_drep; + ib_api_status_t ib_status; + + + // Take the lock and verify the state + rc = m_Lock.Lock(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_Lock.Lock failed rc = 0x%x\n", rc )); + goto Cleanup; + } + + if (m_state != SS_CONNECTED) { + // This is not the state that we waited for, we drop the request + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Not the expacted state %s\n", SS2String(m_state))); + ASSERT(FALSE); // Can this happen on shutdown ? + goto ErrorLocked; + } + + // We should send a DREP now + memset( &cm_drep, 0, sizeof(cm_drep) ); + + ib_status = ib_cm_drep( p_cm_dreq_rec->h_cm_dreq, &cm_drep ); + if( ib_status != IB_SUCCESS ) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_cm_drep failed ib_status = 0x%d\n", ib_status )); + rc = IB2Status(ib_status); + goto ErrorLocked; + } + + // last step is to change our state + m_state = SS_CONNECTED_DREP_SENT; + + // We should close the connection know ??????????/ + + rc = m_Lock.Unlock(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_Lock.Unlock() failed rc = 0x%x\n", rc )); + // BUGBUG: who is responsibale for the cleanup ??????? + } + + //??????? abortive close the connection + +Cleanup: + +/* + if (pSocket != NULL) { + pSocket->Release(); + } +*/ + // Who should take care of the errors that were found here (if found)???????? + return; + +ErrorLocked: + // Previous rc doesn't mater as this function is void + rc = m_Lock.Unlock(); + if (!NT_SUCCESS(rc)) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_Lock.Unlock() failed rc = 0x%x\n", rc )); + // BUGBUG: who is responsibale for the cleanup ??????? + } + goto Cleanup; + +} + VOID SdpSocket::__recv_cb1( @@ -1402,12 +1693,8 @@ SdpSocket::__recv_cb1( NTSTATUS SdpSocket::recv_cb() { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p \n", this)); + SDP_PRINT(SDP_DEBUG, SDP_SOCKET, ("this = 0x%p \n", this)); NTSTATUS rc = STATUS_SUCCESS, rc1 = STATUS_SUCCESS; - if (m_Lock.IsShutdownSignaled()) { - return 0; //?????????????????? this will cause a leak ?????? - } - ib_api_status_t ib_status; ib_wc_t *p_free, *p_wc1; uint32_t pkt_cnt, recv_cnt = 0; @@ -1534,7 +1821,7 @@ static NTSTATUS __recv_cb2(SdpSocket * pSdpSocket) NTSTATUS SdpSocket::send_cb() { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("called this =0x%x\n", this)); + SDP_PRINT(SDP_DEBUG, SDP_SOCKET, ("called this =0x%p\n", this)); NTSTATUS rc = STATUS_SUCCESS, rc1 = STATUS_SUCCESS, rc2 = STATUS_SUCCESS; ib_api_status_t ib_status; ib_wc_t *p_wc, *p_free; @@ -1552,11 +1839,11 @@ NTSTATUS SdpSocket::send_cb() ib_status = ib_poll_cq( m_scq, &p_free, &p_wc ); ASSERT( ib_status == IB_SUCCESS || ib_status == IB_NOT_FOUND); if (ib_status == IB_NOT_FOUND) { - SDP_PRINT(SDP_WARN, SDP_SOCKET, ("ib_poll_cq returned IB_NOT_FOUND, this =0x%x\n", this)); + SDP_PRINT(SDP_WARN, SDP_SOCKET, ("ib_poll_cq returned IB_NOT_FOUND, this =0x%p\n", this)); break; } if (ib_status != IB_SUCCESS) { - SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_poll_cq failed ib_status=%d, this =0x%x\n", ib_status,this)); + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_poll_cq failed ib_status=%d, this =0x%p\n", ib_status,this)); ASSERT(ib_status == IB_INVALID_CQ_HANDLE || ib_status == IB_NOT_FOUND); rc = IB2Status(ib_status); goto Cleanup; @@ -1575,23 +1862,13 @@ NTSTATUS SdpSocket::send_cb() break; case IB_WCS_WR_FLUSHED_ERR: - SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Flushed send completion. this =0x%x\n", this)); + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Flushed send completion. this =0x%p\n", this)); // Intentainly fall down default: SDP_PRINT( SDP_ERR, SDP_SOCKET, ("Send failed with %s\n", ib_get_wc_status_str( p_wc->status )) ); m_Lock.SignalError(IB2Status(ib_status)); } -/* Do we need this ???? - free the memory that was used for the send - if( p_send_buf ) - { - cl_perf_start( FreeSendBuf ); - ExFreeToNPagedLookasideList( &p_port->buf_mgr.send_buf_list, - p_send_buf ); - cl_perf_stop( &p_port->p_adapter->perf, FreeSendBuf ); - } -*/ p_wc = p_wc->p_next; } @@ -1603,7 +1880,6 @@ NTSTATUS SdpSocket::send_cb() rc = rc1; goto Cleanup; } - /* Rearm the CQ. */ ib_status = ib_rearm_cq(m_scq, FALSE ); @@ -1817,8 +2093,8 @@ VOID SdpSocket::CreateHelloHeader( hello_msg->hh.max_adv = QP_ATTRIB_RQ_DEPTH;// ??? conn->l_max_adv; hello_msg->hh.ip_ver = SDP_MSG_IPVER; hello_msg->hh.version = SDP_MSG_VERSION; - hello_msg->hh.r_rcv_size = 4096;//???conn->recv_size; - hello_msg->hh.l_rcv_size = 4096;//???conn->recv_size; + hello_msg->hh.r_rcv_size = MAX_RECV_BUFFER_SIZE;//???conn->recv_size; + hello_msg->hh.l_rcv_size = MAX_RECV_BUFFER_SIZE;//???conn->recv_size; hello_msg->hh.port = m_SrcPort; hello_msg->hh.src.ipv4.addr = m_SrcIp; hello_msg->hh.dst.ipv4.addr = DestIp; @@ -1847,7 +2123,7 @@ VOID SdpSocket::CreateHelloAckHeader( hello_ack_msg->hah.max_adv = QP_ATTRIB_RQ_DEPTH;// ??? conn->l_max_adv; hello_ack_msg->hah.version = SDP_MSG_VERSION; - hello_ack_msg->hah.l_rcv_size = 4096;//???conn->recv_size; + hello_ack_msg->hah.l_rcv_size = MAX_RECV_BUFFER_SIZE;//???conn->recv_size; /* * endian swap @@ -1897,7 +2173,7 @@ VOID SdpSocket::CreateCmRequest( else if( cm_req->local_resp_timeout < CM_MIN_LOCAL_TIMEOUT ) cm_req->local_resp_timeout = CM_MIN_LOCAL_TIMEOUT; - cm_req->rnr_nak_timeout = 6;//6;//???QP_ATTRIB_RNR_NAK_TIMEOUT; + cm_req->rnr_nak_timeout = 6;//???QP_ATTRIB_RNR_NAK_TIMEOUT; cm_req->rnr_retry_cnt = 6;//????QP_ATTRIB_RNR_RETRY; cm_req->retry_cnt = 6;//????QP_ATTRIB_RETRY_COUNT; @@ -1920,31 +2196,64 @@ VOID SdpSocket::ShutdownCB(VOID* pContext) VOID WaitForShutdownEvent(KEVENT *ShutdownCompleteEvent) { NTSTATUS rc = STATUS_SUCCESS; + + LARGE_INTEGER WaitTime; + // Why are we waiting with timeout? obviously we should wait forever + // and if this is the case, there is a bug *SOMEWHERE ELSE* + // Still I wait WAIT_TO_SOCKET_SHUTDOWN seconds and believe that the user + // will like me more If I do quit and let the application close + + WaitTime = TimeFromLong(WAIT_TO_SOCKET_SHUTDOWN * 10 * 1000000); + rc = MyKeWaitForSingleObject( ShutdownCompleteEvent, - UserRequest, - UserMode, + Executive, + KernelMode, FALSE, - NULL + &WaitTime ); + if (rc == STATUS_TIMEOUT) { + SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Wait failed with time out\n")); + ASSERT(FALSE); + } + ASSERT(NT_SUCCESS(rc)); - ASSERT(rc == STATUS_SUCCESS || rc == STATUS_USER_APC); //???????? what to do + ASSERT(rc == STATUS_SUCCESS || rc == STATUS_TIMEOUT);// || rc == STATUS_USER_APC); //???????? what to do //???????? the wait fails KeClearEvent(ShutdownCompleteEvent); } VOID SdpSocket::Shutdown() { - SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Shutdown called this = 0x%p\n", this)); + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("called this = 0x%p\n", this)); NTSTATUS rc = STATUS_SUCCESS; - - // locking ????? - // if(m_shutdown - on the lock) ??? ib_api_status_t ib_status; - - m_Lock.SignalShutdown(); + + if (m_ShutdownCalled) { + // Since this variable is always changing from false to true + // we can check it without the lock + return; + } + // now take the lock and test again + + m_Lock.Lock(); //????????????????????force this locking ???? + if (m_ShutdownCalled) { + // Already handled + m_Lock.Unlock(); // Error is ignored since this is already + // shutdown call + return; + } + m_ShutdownCalled = true; + m_Lock.Unlock(); + + // + // Here we start a list of operations that once comlpeated, + // should force that no complition exists anymore + // Although we are not holding the lock, they will not be able to + // lock it. This means that they will try to take it and fail + if (m_ListenHandle != NULL) { ib_status = ib_cm_cancel( m_ListenHandle, ShutdownCB ); if( ib_status != IB_SUCCESS ) { @@ -1958,38 +2267,66 @@ VOID SdpSocket::Shutdown() m_ConnectionList.Shutdown(); + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("111111\n", this)); if (m_qp != NULL) { ib_status = ib_destroy_qp(m_qp, ShutdownCB); - ASSERT(ib_status == IB_SUCCESS); - m_qp = NULL; - WaitForShutdownEvent(&m_ShutdownCompleteEvent); + //m_qp = NULL; + if(ib_status == IB_SUCCESS) { + WaitForShutdownEvent(&m_ShutdownCompleteEvent); + } else { + ASSERT(ib_status == IB_SUCCESS); + // If this is not the case, then we could have probably being + // closing a different socket, which is very bad. + // To give things a chance to work without shutdown, I'll continue + } } + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("22222\n", this)); + if (m_scq != NULL) { - ib_destroy_cq(m_scq, ShutdownCB); - m_scq = NULL; - WaitForShutdownEvent(&m_ShutdownCompleteEvent); + ib_status = ib_destroy_cq(m_scq, ShutdownCB); + //???ASSERT(ib_status == IB_SUCCESS); + if(ib_status == IB_SUCCESS) { + WaitForShutdownEvent(&m_ShutdownCompleteEvent); + } + //?????m_scq = NULL; } + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("33333\n", this)); if (m_rcq != NULL) { - ib_destroy_cq(m_rcq, ShutdownCB); - m_rcq = NULL; - WaitForShutdownEvent(&m_ShutdownCompleteEvent); + ib_status = ib_destroy_cq(m_rcq, ShutdownCB); + //???ASSERT(ib_status == IB_SUCCESS); + if(ib_status == IB_SUCCESS) { + WaitForShutdownEvent(&m_ShutdownCompleteEvent); + } + //??????m_rcq = NULL; + } - + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("44444\n", this)); if (m_pd != NULL) { - ib_dealloc_pd(m_pd, ShutdownCB); - m_pd = NULL; - WaitForShutdownEvent(&m_ShutdownCompleteEvent); + ib_status = ib_dealloc_pd(m_pd, ShutdownCB); + //???ASSERT(ib_status == IB_SUCCESS); + if(ib_status == IB_SUCCESS) { + WaitForShutdownEvent(&m_ShutdownCompleteEvent); + } + //?????m_pd = NULL; + } + SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("55555\n", this)); if (mh_Ca != NULL) { - ib_close_ca(mh_Ca, ShutdownCB); - mh_Ca = NULL; - WaitForShutdownEvent(&m_ShutdownCompleteEvent); + ib_status = ib_close_ca(mh_Ca, ShutdownCB); + //ASSERT(ib_status == IB_SUCCESS); + if(ib_status == IB_SUCCESS) { + WaitForShutdownEvent(&m_ShutdownCompleteEvent); + } + //?????mh_Ca = NULL; + } + // No compleations should exist any more + if (m_pListeningSocket != NULL) { m_pListeningSocket->Release(); m_pListeningSocket = NULL; diff --git a/trunk/ulp/sdp/kernel/SdpSocket.h b/trunk/ulp/sdp/kernel/SdpSocket.h index 3e03021c..653c9153 100644 --- a/trunk/ulp/sdp/kernel/SdpSocket.h +++ b/trunk/ulp/sdp/kernel/SdpSocket.h @@ -11,24 +11,29 @@ It keeps a list of all the objects so we know when to remove them. #ifndef _SDP_SOCKET_H #define _SDP_SOCKET_H -const int MAX_SEND_BUFFER_SIZE = 32768; // This is the maximum send packet size +const int MAX_SEND_BUFFER_SIZE = 1*4096; // This is the maximum send packet size + +// BUGBUG: Check why changing this param crushes the system +const int MAX_RECV_BUFFER_SIZE = 1*4096; // This is the maximum send packet size const int MAX_SEND_PACKETS = 200; // This is the maximum number of packets allocated per send const int MAX_RECV_PACKETS = 200; // This is the maximum number of packets allocated per send -#define QP_ATTRIB_SQ_DEPTH 64 -#define QP_ATTRIB_SQ_SGE 1 /* Set based on inline data requirements */ +const short QP_ATTRIB_SQ_DEPTH = 64; +const short QP_ATTRIB_SQ_SGE = 1; /* Set based on inline data requirements */ //#define QP_ATTRIB_RESPONDER_RESOURCES 4 -#define QP_ATTRIB_INITIATOR_DEPTH 4 +const short QP_ATTRIB_INITIATOR_DEPTH = 4; //#define QP_ATTRIB_RETRY_COUNT 6 //#define QP_ATTRIB_RNR_RETRY 6 -#define QP_ATTRIB_RNR_NAK_TIMEOUT 6 +const short QP_ATTRIB_RNR_NAK_TIMEOUT = 6; +const short WAIT_TO_SOCKET_SHUTDOWN = 40; // The time to wait for a socket to complete + // it's shutdown before we become brutal /* * TODO: During testing, the switch has been observed to post * 12 receive buffers. It would be nice to know what the max is. */ -#define QP_ATTRIB_RQ_DEPTH 64 +const short QP_ATTRIB_RQ_DEPTH = 64;//64 #define QP_ATTRIB_RQ_SGE 1 const int SDP_RECV_CREDIT_UPDATE = 20; // send credit update to the remote // side. @@ -44,10 +49,12 @@ enum SocketStates { SS_CONNECTED, SS_BOUND, SS_LISTENING, - SS_REQ_RECV, + SS_REQ_RECV, //???? not really used ???? SS_REP_SENT, - SS_INSHUTDOWN + SS_CONNECTED_DREP_SENT //??? not really used }; + // Plesae note that shutdwon and close socket is being done + // through flags void cm_rtu_callback(IN ib_cm_rtu_rec_t *p_cm_rtu_rec ); @@ -70,7 +77,6 @@ private: USHORT m_DstPort; ULONG m_DstIp; - bool m_shutdown; // Make sure this is synced w SdpLock m_Lock; @@ -103,10 +109,16 @@ private: KEVENT m_ConnectCmCompleteEvent; KEVENT m_ShutdownCompleteEvent; + KEVENT m_DisconectSentEvent; ib_wc_t m_SendComplitionWC[QP_ATTRIB_SQ_DEPTH]; ib_wc_t m_RecvComplitionWC[QP_ATTRIB_RQ_DEPTH]; + // The following three falgs are used to shutdown a socket + bool m_CloseSocketCalled; + bool m_ShutdownCalled; + bool m_DisconnectConnectionRecieved; + VOID SignalShutdown(); @@ -163,11 +175,23 @@ public: WspAcceptOut *pWspAcceptOut ); + NTSTATUS WSPCloseSocket( + WspSocketCloseIn *pWspSocketCloseIn, + WspSocketCloseOut *pWspSocketCloseOut + ); VOID Shutdown(); static VOID ShutdownCB(VOID* pContext); + // Make sure that lock does check this ?????????????????? + bool AcceptRequests() { + // Check if our state allows us to handle send/recv/accept ... + if (m_ShutdownCalled) return false; + if (m_CloseSocketCalled) return false; + return true; + } + NTSTATUS CreateQp(); @@ -192,6 +216,12 @@ public: VOID CmRepCallback(IN ib_cm_rep_rec_t *p_cm_rep_rec); VOID CmReqCallback(IN ib_cm_req_rec_t *p_cm_req_rec); VOID CmRtuCallback(IN ib_cm_rtu_rec_t *p_cm_rtu_rec); + VOID CmDreqCallback(IN ib_cm_dreq_rec_t *p_cm_dreq_rec); + + VOID CloseSocketThread(); + + VOID DisconectSentEvent(); + // Two varibales that are needed for passing REP data struct sdp_msg_hello_ack m_hello_ack; @@ -207,10 +237,15 @@ public: char * SS2String(SocketStates state) { switch (state) { case SS_IDLE : return "SS_IDLE"; + case SS_CONNECTING_QPR_SENT : return "SS_CONNECTING_QPR_SENT"; case SS_CONNECTING_REQ_SENT : return "SS_CONNECTING_REQ_SENT"; case SS_CONNECTING_RTU_SENT : return "SS_CONNECTING_RTU_SENT"; case SS_CONNECTED : return "SS_CONNECTED"; - case SS_INSHUTDOWN : return "SS_INSHUTDOWN"; + case SS_BOUND : return "SS_BOUND"; + case SS_LISTENING : return "SS_LISTENING"; + case SS_REQ_RECV : return "SS_REQ_RECV"; + case SS_REP_SENT : return "SS_REP_SENT"; + case SS_CONNECTED_DREP_SENT : return "SS_CONNECTED_DREP_SENT"; default : ASSERT(FALSE); diff --git a/trunk/ulp/sdp/kernel/SdpTrace.cpp b/trunk/ulp/sdp/kernel/SdpTrace.cpp index cba394c2..6a7724d4 100644 --- a/trunk/ulp/sdp/kernel/SdpTrace.cpp +++ b/trunk/ulp/sdp/kernel/SdpTrace.cpp @@ -5,7 +5,8 @@ BOOLEAN CheckCondition(int sev, int top, char *file, int line, char * func) { - if (sev < SDP_WARN) return FALSE; + if (sev < SDP_TRACE) return FALSE; +// if (sev < SDP_WARN) return FALSE; if (top == SDP_PERFORMANCE) return FALSE; DbgPrint ("%s: ", func); diff --git a/trunk/ulp/sdp/kernel/SdpTrace.h b/trunk/ulp/sdp/kernel/SdpTrace.h index 5254c231..8b64b455 100644 --- a/trunk/ulp/sdp/kernel/SdpTrace.h +++ b/trunk/ulp/sdp/kernel/SdpTrace.h @@ -4,7 +4,7 @@ // Debug level masks #define SDP_ALL 0x00000001 -#define SDP_DEBUG 0x00000002 // No - per packet, but very noisy (i.e. credits) +#define SDP_DEBUG 0x00000002 // also per packet, (would probably change) #define SDP_TRACE 0x00000004 // No - per packet (data) print #define SDP_WARN 0x00000008 #define SDP_ERR 0x00000010 diff --git a/trunk/ulp/sdp/kernel/SdpUserFile.cpp b/trunk/ulp/sdp/kernel/SdpUserFile.cpp index 3442dd4c..d675f374 100644 --- a/trunk/ulp/sdp/kernel/SdpUserFile.cpp +++ b/trunk/ulp/sdp/kernel/SdpUserFile.cpp @@ -53,6 +53,26 @@ NTSTATUS SdpUserFile::AddSocket(SdpSocket *pSdpSocket) } +VOID SdpUserFile::RemoveSocket(SdpSocket *pSdpSocket) +{ + CSpinLockWrapper Lock(m_Lock); + Lock.Lock(); + + // If shutodown is running, we don't do anything, weather this + // socket is out of the list already, or that it will soon be + // out + if (m_shutdown) { + Lock.Unlock(); + return ; + } + + RemoveEntryList(&pSdpSocket->m_UserFileList); + pSdpSocket->Release(); + + Lock.Unlock(); +} + + SdpSocket *SdpUserFile::SocketByPointer(VOID *Socket) { SdpSocket *pSdpSocket = NULL; diff --git a/trunk/ulp/sdp/kernel/SdpUserFile.h b/trunk/ulp/sdp/kernel/SdpUserFile.h index 67ab18bf..1e782d3c 100644 --- a/trunk/ulp/sdp/kernel/SdpUserFile.h +++ b/trunk/ulp/sdp/kernel/SdpUserFile.h @@ -28,9 +28,10 @@ public: SdpSocket *SocketByPointer(VOID *Socket); - NTSTATUS AddSocket(SdpSocket *pSdpSocket); + VOID RemoveSocket(SdpSocket *pSdpSocket); + }; #endif //_SDP_USER_FILE_H diff --git a/trunk/ulp/sdp/todo b/trunk/ulp/sdp/todo index 25b433c1..07c1f1b6 100644 --- a/trunk/ulp/sdp/todo +++ b/trunk/ulp/sdp/todo @@ -17,7 +17,7 @@ KERNEL MODE: for the copy recv: - 1) What to do when I don’t have all the buffer to return? + 1) Find and fix the race when the socket is being initialized 2) When posting the credits, consider not sending the credits if there are packets pending and and we have credits. @@ -26,8 +26,6 @@ general: USER MODE: -* check why DHCP doesn't work when the provider is installed. - * Check the lifetime of the SdpSocket (when is it deleted and so)?? * check with intel that we can remove their lisence from the files. @@ -35,4 +33,6 @@ USER MODE: * check the way that errors are reported to the user mode. It seems that returning an error in rc means that the output buffer won't pass out. +* make sure that the "terminator thread" is being killed before we exit. +* Check why sometimes the QP and so are not valid when you come to kill them -- 2.41.0