\r
#include "preCompile.h"\r
\r
+\r
NTSTATUS \r
BufferPool::Init(\r
int MaxBuffers, \r
int MaxMessageSize,\r
ib_pd_handle_t pd,\r
ib_qp_handle_t qp,\r
- net32_t lkey\r
+ net32_t lkey,\r
+ SdpSocket *pSdpSocket\r
+\r
)\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this));\r
m_qp = qp;\r
ASSERT(lkey != NULL);\r
m_lkey = lkey;\r
+#if DBG \r
+ m_pSdpSocket = pSdpSocket;\r
+#endif\r
+\r
\r
return STATUS_SUCCESS;\r
}\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p FirstBuffer = %s\n",this,\r
FirstBuffer ? "TRUE" : "FALSE"));\r
+ AssertLocked();\r
+\r
NTSTATUS rc = STATUS_SUCCESS;\r
*ppBufferDescriptor = NULL;\r
\r
}\r
\r
// Can we supply a buffer right now ?\r
- if (m_CurrentlyAllocated < m_MaxBuffers) {\r
+ if (m_CurrentlySentBuffers < m_MaxConcurrentSends) {\r
// yes, supply a buffer\r
if (m_FreePackets.Size() > 0) {\r
LIST_ENTRY *item = m_FreePackets.RemoveHeadList();\r
*ppBufferDescriptor = CONTAINING_RECORD(item, BufferDescriptor , BuffersList);\r
goto Cleanup;\r
- } else {\r
+ } else if (m_CurrentlyAllocated < m_MaxBuffers) {\r
// we need to alocate a new buffer\r
rc = AllocateBuffer(ppBufferDescriptor);\r
if (!NT_SUCCESS(rc)) {\r
m_CurrentlyAllocated++;\r
goto Cleanup;\r
} \r
- } else {\r
- // No buffers available, we have to wait\r
- ASSERT(m_ClientWaiting == false);\r
- KeClearEvent(&m_WaitingClients);\r
- m_ClientWaiting = true;\r
- *ppEvent = &m_WaitingClients;\r
- }\r
+ } \r
+ // No buffers available, we have to wait\r
+ ASSERT(m_ClientWaiting == false);\r
+ KeClearEvent(&m_WaitingClients);\r
+ m_ClientWaiting = true;\r
+ *ppEvent = &m_WaitingClients;\r
\r
Cleanup: \r
return rc;\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p pBufferDescriptor = 0x%x\n",this,\r
pBufferDescriptor));\r
+ AssertLocked();\r
+\r
NTSTATUS rc = STATUS_SUCCESS;\r
\r
if ((m_CurrentlySentBuffers < m_MaxConcurrentSends) && \r
BufferPool::ReturnBuffer(BufferDescriptor *pBufferDescriptor)\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p buffer=0x%p\n",this, pBufferDescriptor));\r
+\r
+#if DBG \r
+ if (m_CurrentlySentBuffers == 1) {\r
+ SDP_PRINT(SDP_WARN, SDP_BUFFER_POOL, ("Currently no packets are bing sent m_ClientWaiting = %s\n", \r
+ m_ClientWaiting ? "true" : "false")); \r
+ }\r
+#endif \r
+ \r
+ AssertLocked();\r
m_FreePackets.InsertTailList(&pBufferDescriptor->BuffersList);\r
// Is there a client waiting ?\r
if ( m_ClientWaiting) {\r
BufferPool::SendBuffersIfCan()\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this));\r
+ AssertLocked();\r
NTSTATUS rc = STATUS_SUCCESS;\r
\r
while ((m_QueuedPackets.Size() > 0) && \r
BufferPool::ShutDown()\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this));\r
+ //???? AssertLocked();\r
BufferDescriptor *pBufferDescriptor = NULL;\r
LIST_ENTRY *item = NULL;\r
\r
BufferPool::AllocateBuffer(BufferDescriptor ** ppBufferDescriptor)\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this));\r
+ AssertLocked();\r
NTSTATUS rc = STATUS_SUCCESS;\r
BufferDescriptor *pBufferDescriptor = NULL;\r
\r
ExFreePoolWithTag(pBufferDescriptor->pBuffer, SEND_BUFFERS_ALLOCATION_TAG);\r
}\r
ExFreePoolWithTag(pBufferDescriptor, SEND_BUFFERS_ALLOCATION_TAG);\r
+ pBufferDescriptor = NULL;\r
} \r
}\r
*ppBufferDescriptor = pBufferDescriptor;\r
BufferPool::SendBuffer(BufferDescriptor *pBufferDescriptor)\r
{\r
SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n",this));\r
+ AssertLocked();\r
NTSTATUS rc = STATUS_SUCCESS;\r
\r
msg_hdr_bsdh *pHeader = (msg_hdr_bsdh *) pBufferDescriptor->pBuffer;\r
return rc;\r
}\r
\r
+VOID \r
+BufferPool::AssertLocked() {\r
+#if DBG\r
+ m_pSdpSocket->AssertLocked();\r
+#endif\r
+}\r
+\r
\r
#define H_SDP_BUFFER_POOL_H \r
\r
\r
+\r
// This is simply a wrapper to the LIST_ENTRY class that allows \r
// easier work with this list\r
class LinkedList {\r
int MaxMessageSize,\r
ib_pd_handle_t pd,\r
ib_qp_handle_t qp,\r
- net32_t lkey\r
+ net32_t lkey,\r
+ SdpSocket *pSdpSocket\r
);\r
\r
NTSTATUS GetBuffer(\r
bool m_ClientBeingServed; // true if we have already started giving buffers to a client\r
\r
LinkedList m_FreePackets; // This packets are free and might be used\r
- LinkedList m_QueuedPackets; // This packets were filled with data and should be filled\r
+ LinkedList m_QueuedPackets; // This packets were filled with data and should be sent\r
\r
\r
// TODO: A queue of events for threads that are waiting for buffers.\r
KEVENT m_WaitingClients; // switch to a linked list\r
bool m_ClientWaiting;\r
\r
+#if DBG\r
+ SdpSocket *m_pSdpSocket;\r
+#endif //DBG\r
+\r
+VOID AssertLocked();\r
+\r
};\r
\r
#endif // H_SDP_BUFFER_POOL_H\r
&DevName,\r
FILE_DEVICE_UNKNOWN,\r
0, \r
- TRUE,\r
+ FALSE,\r
&pDevObj );\r
if (!NT_SUCCESS(rc)) {\r
SDP_PRINT(SDP_ERR, SDP_DRIVER, ("IoCreateDevice failed rc = 0x%x\n", rc )); \r
KeInitializeEvent(&m_Event, NotificationEvent , TRUE);\r
KeInitializeSpinLock(&m_SpinLock);\r
m_SendCBHandler = NULL;\r
+ m_ClientWaiting = false;\r
}\r
\r
VOID Init(SendCBHandler SendCB, SdpSocket *pSdpSocket)\r
\r
if (m_InUse) {\r
// We have to release the spinlock and wait on the event\r
+ m_ClientWaiting = true;\r
KeReleaseSpinLock(&m_SpinLock, OldIrql);\r
rc = MyKeWaitForSingleObject(&m_Event, UserRequest, UserMode, false, NULL);\r
if (( rc == STATUS_ALERTED ) ||( rc == STATUS_USER_APC )) {\r
KeClearEvent(&m_Event);\r
OldFlags = m_flags;\r
ResetFlags(m_flags);\r
+ m_ClientWaiting = false;\r
KeReleaseSpinLock(&m_SpinLock, OldIrql);\r
rc = HandleFlags(OldFlags);\r
if (!NT_SUCCESS(rc)) {\r
}\r
KeReleaseSpinLock(&m_SpinLock, OldIrql);\r
if (SomethingToHandle(OldFlags)) {\r
+ ASSERT(m_InUse);\r
rc = HandleFlags(OldFlags); \r
if (!NT_SUCCESS(rc)) {\r
// We have to signal the error to the calling side\r
NTSTATUS rc = STATUS_SUCCESS;\r
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);\r
KeAcquireSpinLock(&m_SpinLock, &OldIrql);\r
- if (m_InUse) {\r
+ if (m_InUse || m_ClientWaiting ) {\r
m_flags |= flags;\r
KeReleaseSpinLock(&m_SpinLock, OldIrql);\r
return false;\r
*/\r
NTSTATUS HandleFlags(int flags) {\r
NTSTATUS rc = STATUS_SUCCESS;\r
+ AssertLocked();\r
if (flags & SEND_CB_CALLED) {\r
// We need to handle the send CB\r
rc = m_SendCBHandler(m_pSdpSocket);\r
return rc;\r
}\r
\r
- VOID SignalShutdown() {ASSERT (FALSE);} //????????????? Make sure this is used\r
- VOID SignalError(NTSTATUS rc) {ASSERT (FALSE);} //????????????? \r
+ VOID SignalShutdown() {\r
+ //??????? Verify use and correctnes\r
+ m_flags |= SHUTDOWN_SIGNALLED;\r
+ }\r
+\r
+ bool IsShutdownSignaled()\r
+ {\r
+ return m_flags & SHUTDOWN_SIGNALLED ? true : false;\r
+ }\r
+ \r
+ VOID SignalError(NTSTATUS rc) {ASSERT (FALSE);} //????????????\r
+\r
+ VOID AssertLocked() {ASSERT(m_InUse);}\r
\r
- KEVENT m_Event; // the event for passive level threads\r
- KSPIN_LOCK m_SpinLock; // The real guard of the lock\r
+ KEVENT m_Event; // the event for passive level threads\r
+ KSPIN_LOCK m_SpinLock; // The real guard of the lock\r
SendCBHandler m_SendCBHandler;\r
\r
\r
- bool m_InUse; // Tells if this lock has any user\r
- int m_flags; // call backs that were recieved\r
+ bool m_InUse; // Tells if this lock has any user\r
+ int m_flags; // call backs that were recieved\r
+ bool m_ClientWaiting; // True if there is a client waiting to be served\r
\r
- SdpSocket *m_pSdpSocket; // The socket that this class depends on\r
+ SdpSocket *m_pSdpSocket;// The socket that this class depends on\r
};\r
\r
#endif // _SDP_LOCK_H\r
m_state = SS_IDLE;\r
}\r
\r
+VOID SdpSocket::AssertLocked()\r
+{\r
+#if DBG\r
+ m_Lock.AssertLocked();\r
+#endif\r
+}\r
+\r
NTSTATUS SdpSocket::Init(\r
WspSocketIn *pSocketInParam, \r
WspSocketOut *pSocketOutParam)\r
goto Cleanup;\r
}\r
Locked = true;\r
+ ASSERT(pBuffersEvent == NULL);\r
\r
rc = m_SendBufferPool.GetBuffer(&pBufferDescriptor, &pBuffersEvent, First);\r
if (!NT_SUCCESS(rc)) {\r
FALSE,\r
NULL\r
); \r
-\r
+ pBuffersEvent = NULL;\r
if (( rc == STATUS_ALERTED ) ||( rc == STATUS_USER_APC )) {\r
// BUGBUG: Think what to do here, we should be able to stop the\r
// connect, and quit (probably shutdown should be enough)\r
ULONG CopySize = pBufferDescriptor->BufferSize - sizeof msg_hdr_bsdh;\r
CopySize = min(CopySize, pWspSendIn->BufferSize - Coppied);\r
\r
- pBufferDescriptor->WriteData(pWspSendIn->pData + Coppied, CopySize);\r
+ rc = pBufferDescriptor->WriteData(pWspSendIn->pData + Coppied, CopySize);\r
+ if (!NT_SUCCESS(rc)) {\r
+ SDP_PRINT(SDP_ERR, SDP_SOCKET, ("pBufferDescriptor->WriteData failed rc = 0x%x\n", rc ));\r
+ // free the buffer that you have\r
+ m_SendBufferPool.ReturnBuffer(pBufferDescriptor);\r
+ m_Lock.Unlock(); // Error ignored as this is already an error pass \r
+ goto Cleanup;\r
+ }\r
Coppied += CopySize;\r
\r
// return the data to the buffer\r
return rc;\r
}\r
\r
+\r
#if 0\r
//Naive send implmentation.\r
NTSTATUS SdpSocket::WSPSend(\r
\r
int MaxMessageSize = min(m_hello_ack.hah.l_rcv_size, MAX_SEND_BUFFER_SIZE);\r
\r
- rc = m_SendBufferPool.Init(MAX_SEND_PACKETS, QP_ATTRIB_SQ_DEPTH, MaxMessageSize, m_pd, m_qp, m_lkey);\r
+ rc = m_SendBufferPool.Init(MAX_SEND_PACKETS, QP_ATTRIB_SQ_DEPTH, MaxMessageSize, m_pd, m_qp, m_lkey, this);\r
if (!NT_SUCCESS(rc)) {\r
SDP_PRINT(SDP_ERR, SDP_SOCKET, ("m_SendBufferPool.Init failed rc = 0x%x\n", rc ));\r
goto Cleanup;\r
goto Cleanup;\r
}\r
\r
+ // We now start the recieve processing\r
+ RecieveOnce();\r
+\r
Cleanup: \r
return rc;\r
}\r
}\r
\r
\r
+//????????? Let's implmeant a naive read\r
// BUGBUG: based on __recv_cb - need to implment\r
static void\r
__recv_cb1(\r
- IN const ib_cq_handle_t h_cq,\r
- IN void *cq_context )\r
+ IN const ib_cq_handle_t h_cq,\r
+ IN void *cq_context )\r
{\r
-// status = ib_rearm_cq(\r
-// p_port->ib_mgr.h_recv_cq, FALSE );\r
-// ASSERT(exp)( status == IB_SUCCESS );\r
- ASSERT(FALSE);\r
+ SdpSocket *pSocket = (SdpSocket *) cq_context;\r
+ //pSocket->m_Lock.SignalCB(RECV_CB_CALLED);\r
+\r
+ pSocket->__recv_cb2();\r
+ \r
}\r
+char g_Recieve[6000];\r
+\r
+VOID \r
+SdpSocket::__recv_cb2()\r
+{\r
+ if (m_Lock.IsShutdownSignaled()) {\r
+ return;\r
+ }\r
+\r
+ ib_api_status_t status;\r
+ ib_wc_t wc[QP_ATTRIB_RQ_DEPTH], *p_free, *p_wc1;\r
+ uint32_t pkt_cnt, recv_cnt = 0;\r
+ size_t i;\r
+\r
+\r
+ for( i = 0; i < QP_ATTRIB_RQ_DEPTH; i++ )\r
+ wc[i].p_next = &wc[i + 1];\r
+ wc[QP_ATTRIB_RQ_DEPTH - 1].p_next = NULL;\r
+\r
+ do\r
+ {\r
+ /* If we get here, then the list of WCs is intact. */\r
+ p_free = wc;\r
+\r
+ status = ib_poll_cq( m_rcq, &p_free, &p_wc1 );\r
+ CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
+\r
+ /* Look at the payload now and filter ARP and DHCP packets. */\r
+ //recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
+ if (status == IB_SUCCESS)\r
+ {\r
+ ib_wc_t *p_wc;\r
+ for( p_wc = p_wc1; p_wc; p_wc = p_wc->p_next ) {\r
+ ASSERT( p_wc->status == IB_WCS_SUCCESS );\r
+ int len = p_wc->length;\r
+ ASSERT(len >= sizeof msg_hdr_bsdh);\r
+ msg_hdr_bsdh *pHeader = (msg_hdr_bsdh *)g_Recieve;\r
+ sdp_msg_swap_bsdh(pHeader);\r
+ ASSERT(pHeader->mid == 0xff && pHeader->size == 0x10);\r
+ }\r
+ }\r
+\r
+ } while( !p_free );\r
+\r
+\r
+ /* Update our posted depth. */\r
+\r
+ /* Notify NDIS of any and all possible receive buffers. */\r
+\r
+ /* Only indicate receives if we actually had any. */\r
+// if( pkt_cnt )\r
+ {\r
+// NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
+// p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
+ }\r
+\r
+ /* Return any discarded receives to the pool */\r
+\r
+ /* Repost receives. */\r
+\r
+ RecieveOnce();\r
+\r
+\r
+ /*\r
+ * Rearm after filtering to prevent contention on the enpoint maps\r
+ * and eliminate the possibility of having a call to\r
+ * __endpt_mgr_insert find a duplicate.\r
+ */\r
+ status = ib_rearm_cq(m_rcq, FALSE );\r
+ CL_ASSERT( status == IB_SUCCESS );\r
+\r
+\r
+\r
+}\r
+\r
+\r
+\r
+VOID \r
+SdpSocket::RecieveOnce()\r
+{\r
+ ib_local_ds_t l_ds;\r
+\r
+ ib_recv_wr_t recv_wr;\r
+ recv_wr.p_next = NULL;\r
+ recv_wr.num_ds = 1;\r
+ recv_wr.wr_id = 123;\r
+ recv_wr.ds_array = & l_ds;\r
+\r
+ l_ds.length = sizeof(g_Recieve);\r
+ l_ds.lkey = m_lkey;\r
+ l_ds.vaddr = MmGetPhysicalAddress( g_Recieve ).QuadPart;\r
+ \r
+ ib_api_status_t ib_status = ib_post_recv(m_qp, &recv_wr, NULL);\r
+ ASSERT(ib_status == IB_SUCCESS);\r
+ \r
+}\r
+\r
+\r
\r
// TODO: Clear the callback functions mess\r
void\r
p_free = wc;\r
ib_status = ib_poll_cq( m_scq, &p_free, &p_wc );\r
ASSERT( ib_status == IB_SUCCESS || ib_status == IB_NOT_FOUND);\r
+ if (ib_status == IB_NOT_FOUND) {\r
+ SDP_PRINT(SDP_WARN, SDP_SOCKET, ("ib_poll_cq returned IB_NOT_FOUND, this =0x%x\n", this));\r
+ break;\r
+ }\r
if (ib_status != IB_SUCCESS) { \r
SDP_PRINT(SDP_ERR, SDP_SOCKET, ("ib_poll_cq failed ib_status=%d, this =0x%x\n", ib_status,this));\r
ASSERT(ib_status == IB_INVALID_CQ_HANDLE || ib_status == IB_NOT_FOUND);\r
\r
VOID SdpSocket::Shutdown()\r
{\r
+ \r
//???? locking\r
// if(m_shutdown - on the lock) ???\r
ib_api_status_t ib_status;\r
-\r
+ m_Lock.SignalShutdown();\r
\r
SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("SdpSocket::Shutdown called this = 0x%p\n", this));\r
\r
#define _SDP_SOCKET_H\r
\r
const int MAX_SEND_BUFFER_SIZE = 32768; // This is the maximum send packet size\r
-const int MAX_SEND_PACKETS = 40; // This is the maximum number of packets allocated per send\r
+const int MAX_SEND_PACKETS = 800; // This is the maximum number of packets allocated per send\r
\r
\r
\r
-#define QP_ATTRIB_SQ_DEPTH 16\r
+#define QP_ATTRIB_SQ_DEPTH 32\r
#define QP_ATTRIB_SQ_SGE 1 /* Set based on inline data requirements */\r
\r
/* \r
// Used to allow the user file to remember us\r
LIST_ENTRY m_UserFileList;\r
\r
-#ifdef DBG\r
+#if DBG\r
char * SS2String(SocketStates state) {\r
switch (state) {\r
case SS_IDLE : return "SS_IDLE";\r
}\r
return "Unknown state";\r
}\r
+\r
#endif\r
+\r
+ VOID AssertLocked();\r
+\r
+ VOID RecieveOnce(); //???????? remove me\r
+ VOID __recv_cb2();\r
\r
};\r
\r
\r
BOOLEAN CheckCondition(int sev, int top, char *file, int line, char * func)\r
{\r
-// return FALSE;\r
+ if (sev < SDP_WARN) return FALSE;\r
DbgPrint ("%s: ", func);\r
+ if (sev == SDP_ERR) DbgPrint ("ERROR - ");\r
return TRUE;\r
}
\ No newline at end of file
\r
BOOLEAN CheckCondition(int sev, int top, char *file, int line, char * func);\r
\r
+#if DBG\r
#define SDP_PRINT(sev, toppic ,var_args) \\r
if (CheckCondition(sev, toppic ,__FILE__, __LINE__,__FUNCTION__) != FALSE) \\r
DbgPrint var_args\r
-\r
+#else\r
+#define SDP_PRINT(sev, toppic ,var_args)\r
+#endif\r
\r
\r
#endif //H_SDP_TRACE_H\r
=============================================================================\r
\r
KERNEL MODE:\r
-1) On send: implmeant some kind of a negal algorithm.\r
-2) On send: Create some kind of mechanism that will allow to recieve complitions on more than\r
-one send.\r
-\r
+ send:\r
+ 1) On send: implmeant some kind of a negal algorithm.\r
+ 2) On send: Create some kind of mechanism that will allow to recieve complitions on more than\r
+ one send.\r
+ 3) If possibale, post more than one send.\r
+ 4) \r
\r
\r
\r
projects / ~shefty / rdma-win.git / commitdiff