switch( socket_info->socket_state )\r
{\r
case IBSP_LISTEN:\r
- if( cl_qlist_count( &socket_info->info.listen.list ) >= socket_info->info.listen.backlog )\r
+ if( cl_qlist_count( &socket_info->listen.list ) >=\r
+ socket_info->listen.backlog )\r
{\r
/* Already too many connection requests are queued */\r
IBSP_TRACE1( IBSP_DBG_CM,\r
}\r
\r
/* Add to the waiting list */\r
- cl_qlist_insert_head( &socket_info->info.listen.list, &incoming->item );\r
+ cl_qlist_insert_tail( &socket_info->listen.list, &incoming->item );\r
\r
/* Send MRA */\r
mra.mra_length = 0;\r
wait_cq_drain( socket_info );\r
\r
/* Non-blocking cancel since we're in CM callback context */\r
- ib_cm_cancel( socket_info->info.listen.handle, NULL );\r
- socket_info->info.listen.handle = NULL;\r
+ ib_cm_cancel( socket_info->listen.handle, NULL );\r
+ socket_info->listen.handle = NULL;\r
\r
#if 0\r
cl_spinlock_release( &socket_info->mutex );\r
\r
cl_spinlock_acquire( &socket_info->mutex );\r
\r
- if( socket_info->socket_state == IBSP_CONNECT )\r
+ switch( socket_info->socket_state )\r
{\r
+ case IBSP_CONNECT:\r
status = ib_cm_rtu( p_cm_rep_rec->h_cm_rep, &cm_rtu );\r
if( status != IB_SUCCESS )\r
{\r
/* Note: a REJ has been automatically sent. */\r
CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("ib_cm_rtu failed (0x%d)\n", status) );\r
IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_BIND );\r
- ibsp_post_select_event( socket_info, FD_CONNECT, WSAENOBUFS );\r
+ ibsp_post_select_event( socket_info, FD_CONNECT, WSAETIMEDOUT );\r
}\r
else\r
{\r
IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CONNECTED );\r
ibsp_post_select_event( socket_info, FD_CONNECT, 0 );\r
}\r
- }\r
- else if( socket_info->socket_state == IBSP_DUPLICATING_NEW )\r
- {\r
+ break;\r
+\r
+ case IBSP_DUPLICATING_NEW:\r
status = ib_cm_rtu( p_cm_rep_rec->h_cm_rep, &cm_rtu );\r
if( status != IB_SUCCESS )\r
{\r
{\r
IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CONNECTED );\r
}\r
- SetEvent( socket_info->info.connect.event );\r
- }\r
- else\r
- {\r
+ SetEvent( socket_info->h_event );\r
+\r
+ default:\r
/* The socket might be closing */\r
CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("socket %x not in connecting state (%s)\n",\r
socket_info, IBSP_SOCKET_STATE_STR( socket_info->socket_state )) );\r
if( socket_info->socket_state == IBSP_ACCEPT )\r
{\r
IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CONNECTED );\r
- SetEvent( socket_info->info.accept.event );\r
+ SetEvent( socket_info->h_event );\r
}\r
else if( socket_info->socket_state == IBSP_DUPLICATING_REMOTE )\r
{\r
\r
status = ib_post_recv( socket_info->qp, &wr->recv, NULL );\r
\r
-\r
if( status == IB_SUCCESS )\r
{\r
/* Update the index and wrap as needed */\r
case IBSP_DUPLICATING_NEW:\r
/* Leave in that state. IBSPSocket will eventually return \r
* an error becaus the socket is not connected. */\r
- SetEvent( socket_info->info.connect.event );\r
+ SetEvent( socket_info->h_event );\r
break;\r
\r
default:\r
socket_info->duplicate.dwProcessId = reason->duplicating.dwProcessId;\r
\r
/* Now, setup our listening callback. */\r
- socket_info->info.listen.listen_req_param.dwProcessId =\r
+ socket_info->listen.listen_req_param.dwProcessId =\r
reason->duplicating.dwProcessId;\r
- socket_info->info.listen.listen_req_param.identifier =\r
+ socket_info->listen.listen_req_param.identifier =\r
reason->duplicating.identifier;\r
\r
ret = ib_listen( socket_info, &error );\r
socket_info->qp_error = WSAECONNRESET;\r
}\r
}\r
- else\r
- {\r
- }\r
cl_spinlock_release( &socket_info->mutex );\r
\r
memset( &cm_drep, 0, sizeof(cm_drep) );\r
}\r
param.lid = IB_ALL_LIDS;\r
\r
- param.p_compare_buffer = (uint8_t *) & socket_info->info.listen.listen_req_param;\r
+ param.p_compare_buffer = (uint8_t *) & socket_info->listen.listen_req_param;\r
param.compare_length = sizeof(struct listen_req_param);\r
param.compare_offset = offsetof(struct cm_req_params, listen_req_param);\r
\r
fzprint(("%s():%d:0x%x:0x%x: socket=0x%p params: %x %x\n", __FUNCTION__,\r
__LINE__, GetCurrentProcessId(),\r
GetCurrentThreadId(), socket_info,\r
- socket_info->info.listen.listen_req_param.dwProcessId,\r
- socket_info->info.listen.listen_req_param.identifier));\r
+ socket_info->listen.listen_req_param.dwProcessId,\r
+ socket_info->listen.listen_req_param.identifier));\r
\r
param.pfn_cm_req_cb = cm_req_callback;\r
\r
param.qp_type = IB_QPT_RELIABLE_CONN;\r
\r
status = ib_cm_listen( g_ibsp.al_handle, ¶m, listen_err_callback, socket_info, /* context */\r
- &socket_info->info.listen.handle );\r
+ &socket_info->listen.handle );\r
\r
if( status != IB_SUCCESS )\r
{\r
cl_list_item_t *item;\r
struct listen_incoming *incoming;\r
\r
- socket_info->info.listen.backlog = backlog;\r
+ socket_info->listen.backlog = backlog;\r
\r
while(\r
- cl_qlist_count( &socket_info->info.listen.list ) > (uint32_t)backlog )\r
+ cl_qlist_count( &socket_info->listen.list ) > (uint32_t)backlog )\r
{\r
- item = cl_qlist_remove_tail( &socket_info->info.listen.list );\r
+ item = cl_qlist_remove_tail( &socket_info->listen.list );\r
\r
incoming = PARENT_STRUCT(item, struct listen_incoming, item);\r
\r
\r
IBSP_ENTER( IBSP_DBG_CM );\r
\r
- /* We should be in the CLOSING state */\r
- CL_ASSERT( socket_info->socket_state == IBSP_CLOSING );\r
-\r
- status = ib_cm_cancel( socket_info->info.listen.handle, ib_sync_destroy );\r
+ status = ib_cm_cancel( socket_info->listen.handle, ib_sync_destroy );\r
if( status )\r
{\r
CL_ERROR( IBSP_DBG_CM, gdbg_lvl,\r
ib_listen_backlog( socket_info, 0 );\r
cl_spinlock_release( &socket_info->mutex );\r
\r
- socket_info->info.listen.handle = NULL;\r
+ socket_info->listen.handle = NULL;\r
\r
IBSP_EXIT( IBSP_DBG_CM );\r
}\r
}\r
\r
IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_DUPLICATING_NEW );\r
- socket_info->info.connect.event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
+ socket_info->h_event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
\r
/* Connects the QP. */\r
ret = ib_connect( socket_info, &path_rec );\r
goto done;\r
}\r
\r
- if( WaitForSingleObject( socket_info->info.connect.event, INFINITE ) != WAIT_OBJECT_0 )\r
+ if( WaitForSingleObject( socket_info->h_event, INFINITE ) != WAIT_OBJECT_0 )\r
{\r
CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
}\r
if( h )\r
CloseHandle( h );\r
\r
- if( socket_info->info.connect.event )\r
+ if( socket_info->h_event )\r
{\r
- CloseHandle( socket_info->info.connect.event );\r
- socket_info->info.connect.event = NULL;\r
+ CloseHandle( socket_info->h_event );\r
+ socket_info->h_event = NULL;\r
}\r
\r
CL_EXIT( IBSP_DBG_CONN, gdbg_lvl );\r
\r
typedef struct _io_comp_info\r
{\r
- SOCKET socket;\r
+ struct ibsp_socket_info *p_socket;\r
LPWSAOVERLAPPED p_ov;\r
- atomic32_t *p_io_cnt;\r
\r
} io_comp_info_t;\r
\r
CL_ASSERT( wr );\r
\r
socket_info = wr->socket_info;\r
- p_io_info->socket = socket_info->switch_socket;\r
+ p_io_info->p_socket = socket_info;\r
\r
lpOverlapped = wr->lpOverlapped;\r
\r
\r
cl_atomic_inc( &socket_info->dup_cnt );\r
/* ib_cq_comp will decrement the receive count. */\r
- p_io_info->p_io_cnt = &socket_info->recv_cnt;\r
+ cl_atomic_dec( &socket_info->recv_cnt );\r
\r
cl_spinlock_release( &socket_info->recv_lock );\r
\r
lpOverlapped->InternalHigh, lpOverlapped->hEvent) );\r
\r
p_io_info->p_ov = lpOverlapped;\r
+ cl_atomic_inc( &socket_info->ref_cnt );\r
}\r
\r
if( wc->wc_type == IB_WC_RECV )\r
- p_io_info->p_io_cnt = &socket_info->recv_cnt;\r
+ cl_atomic_dec( &socket_info->recv_cnt );\r
else\r
- p_io_info->p_io_cnt = &socket_info->send_cnt;\r
+ cl_atomic_dec( &socket_info->send_cnt );\r
\r
IBSP_EXIT( IBSP_DBG_IO );\r
}\r
done_wclist = done_wclist->p_next;\r
}\r
\r
- while( cb_idx-- )\r
+ for( i = 0; i < cb_idx; i++ )\r
{\r
int error;\r
int ret;\r
\r
- if( info[cb_idx].p_ov )\r
+ if( info[i].p_ov )\r
{\r
+ IBSP_TRACE1( IBSP_DBG_IO,\r
+ ("Calling WPUCompleteOverlappedRequest for ov=%p\n",\r
+ info[i].p_ov) );\r
+\r
ret = g_ibsp.up_call_table.lpWPUCompleteOverlappedRequest(\r
- info[cb_idx].socket, info[cb_idx].p_ov,\r
- info[cb_idx].p_ov->OffsetHigh,\r
- (DWORD)info[cb_idx].p_ov->InternalHigh, &error );\r
+ info[i].p_socket->switch_socket, info[i].p_ov,\r
+ info[i].p_ov->OffsetHigh,\r
+ (DWORD)info[i].p_ov->InternalHigh, &error );\r
if( ret != 0 )\r
{\r
IBSP_ERROR( ("WPUCompleteOverlappedRequest for ov=%p "\r
- "returned %d err %d\n", info[cb_idx].p_ov, ret, error) );\r
+ "returned %d err %d\n", info[i].p_ov, ret, error) );\r
}\r
+ deref_socket_info( info[i].p_socket );\r
}\r
-\r
- cl_atomic_dec( info[cb_idx].p_io_cnt );\r
}\r
\r
#ifdef _DEBUG_\r
\r
unregister_pnp();\r
\r
- if( g_ibsp.ib_cleanup_thread )\r
- {\r
- /* Let thread know it's okay to exit after resources are freed */\r
- g_ibsp.ib_cleanup_thread_exit_wanted = TRUE;\r
- SetEvent( g_ibsp.ib_cleanup_event );\r
-\r
- fzprint(("%s():%d:0x%x:0x%x: Waiting for ib_cleanup_thread to die.\n",\r
- __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
- GetCurrentThreadId()));\r
-\r
- /* Wait for ib_cleanup_thread to die */\r
- if( WaitForSingleObject( g_ibsp.ib_cleanup_thread, INFINITE ) != WAIT_OBJECT_0 )\r
- {\r
- CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
- }\r
- else\r
- {\r
- STAT_DEC( thread_num );\r
- }\r
-\r
- fzprint(("%s():%d:0x%x:0x%x: ib_cleanup_thread exited.\n", __FUNCTION__,\r
- __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
- CloseHandle( g_ibsp.ib_cleanup_thread );\r
- g_ibsp.ib_cleanup_thread = NULL;\r
- }\r
-\r
- if( g_ibsp.ib_cleanup_event )\r
- {\r
- CloseHandle( g_ibsp.ib_cleanup_event );\r
- g_ibsp.ib_cleanup_event = NULL;\r
- }\r
-\r
while( (item = cl_qlist_head( &g_ibsp.hca_list )) != cl_qlist_end( &g_ibsp.hca_list ) )\r
{\r
struct ibsp_hca *hca = PARENT_STRUCT(item, struct ibsp_hca, item);\r
}\r
\r
\r
-/* IP notify thread */\r
-static DWORD WINAPI\r
-ib_cleanup_thread(\r
- LPVOID lpParameter )\r
-{\r
- cl_list_item_t *socket_item = NULL;\r
-\r
- CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
-\r
- UNUSED_PARAM( lpParameter );\r
-\r
- while( !g_ibsp.ib_cleanup_thread_exit_wanted ||\r
- cl_qlist_count( &g_ibsp.socket_info_list ) )\r
- {\r
- if( g_ibsp.ib_cleanup_thread_exit_wanted == FALSE )\r
- {\r
- if( WaitForSingleObject( g_ibsp.ib_cleanup_event, INFINITE ) != WAIT_OBJECT_0 )\r
- {\r
- CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
- }\r
- ResetEvent( g_ibsp.ib_cleanup_event );\r
- }\r
- else\r
- {\r
- fzprint(("%s():%d:0x%x:0x%x: socket_info_list cnt=%d\n", __FUNCTION__,\r
- __LINE__, GetCurrentProcessId(),\r
- GetCurrentThreadId(),\r
- cl_qlist_count( &g_ibsp.socket_info_list) == 0));\r
- Sleep( 1000 );\r
- }\r
-\r
- CL_TRACE( IBSP_DBG_WQ, gdbg_lvl, ("%s():%d:0x%x:0x%x: Wakeup\n",\r
- __FUNCTION__,\r
- __LINE__, GetCurrentProcessId(),\r
- GetCurrentThreadId()));\r
-\r
- cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
- while( (socket_item = cl_qlist_remove_head( &g_ibsp.closed_socket_info_list )) !=\r
- cl_qlist_end( &g_ibsp.closed_socket_info_list ) )\r
- {\r
- struct ibsp_socket_info *socket_info = NULL;\r
-\r
- cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
-\r
- socket_info = PARENT_STRUCT(socket_item, struct ibsp_socket_info, item);\r
-\r
-#ifdef _DEBUG_\r
- {\r
- uint8_t idx, i;\r
- LPOVERLAPPED lpOverlapped;\r
-\r
- idx = socket_info->send_idx - (uint8_t)socket_info->send_cnt;\r
- if( idx >= QP_ATTRIB_SQ_DEPTH )\r
- idx += QP_ATTRIB_SQ_DEPTH;\r
-\r
- for( i = 0; i < socket_info->send_cnt; i++ )\r
- {\r
- lpOverlapped = socket_info->send_wr[idx].lpOverlapped;\r
- fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
- __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->send_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
-\r
- if( ++idx == QP_ATTRIB_SQ_DEPTH )\r
- idx = 0;\r
- }\r
-\r
- idx = socket_info->recv_idx - (uint8_t)socket_info->recv_cnt;\r
- if( idx >= QP_ATTRIB_RQ_DEPTH )\r
- idx += QP_ATTRIB_RQ_DEPTH;\r
-\r
- for( i = 0; i < socket_info->recv_cnt; i++ )\r
- {\r
- lpOverlapped = socket_info->recv_wr[idx].wr.lpOverlapped;\r
- fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
- __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->recv_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
-\r
- if( ++idx == QP_ATTRIB_RQ_DEPTH )\r
- idx = 0;\r
- }\r
- }\r
-#endif\r
- fzprint(("%s():%d:0x%x:0x%x: socket=0x%p\n",\r
- __FUNCTION__,\r
- __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info));\r
-\r
- wait_cq_drain( socket_info );\r
-\r
- if( socket_info->dup_cnt )\r
- ibsp_dup_overlap_abort( socket_info );\r
-\r
- /* Destroy the switch socket. */\r
- if( socket_info->switch_socket != INVALID_SOCKET )\r
- {\r
- int ret;\r
- int error;\r
-\r
- fzprint(("%s():%d:0x%x:0x%x: socket=0x%p calling lpWPUCloseSocketHandle=0x%p\n", __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, socket_info->switch_socket));\r
-\r
- ret = g_ibsp.up_call_table.lpWPUCloseSocketHandle(\r
- socket_info->switch_socket, &error );\r
- if( ret == SOCKET_ERROR )\r
- {\r
- CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
- ("WPUCloseSocketHandle failed: %d\n", error) );\r
- }\r
- else\r
- {\r
- STAT_DEC( wpusocket_num );\r
- }\r
-\r
- socket_info->switch_socket = INVALID_SOCKET;\r
- }\r
-\r
- ib_destroy_socket( socket_info );\r
-\r
- ib_deregister_all_mr( &socket_info->buf_mem_list );\r
- free_socket_info( socket_info );\r
- cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
- }\r
- cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
- }\r
-\r
- /* No special exit code, even on errors. */\r
- CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
- ExitThread( 0 );\r
-}\r
-\r
-\r
-\r
/* Initialize IB ressources. */\r
int\r
ibsp_initialize(void)\r
/* Populate IP list. */\r
update_all_ip_addrs();\r
\r
- /* Create a cleanup event */\r
- g_ibsp.ib_cleanup_event = CreateEvent( NULL, TRUE, FALSE, NULL );\r
- if( g_ibsp.ib_cleanup_event == NULL )\r
- {\r
- CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("CreateEvent failed."));\r
- ret = WSAEPROVIDERFAILEDINIT;\r
- goto done;\r
- }\r
-\r
- /* Create a cleanup thread */\r
- g_ibsp.ib_cleanup_thread = CreateThread( NULL, 0, ib_cleanup_thread, NULL, 0, NULL );\r
-\r
- if( g_ibsp.ib_cleanup_thread == NULL )\r
- {\r
- CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("CreateThread failed.") );\r
- ret = WSAEPROVIDERFAILEDINIT;\r
- goto done;\r
- }\r
-\r
STAT_INC( thread_num );\r
\r
ret = 0;\r
/* Closes a connection and release its ressources. */\r
void\r
shutdown_and_destroy_socket_info(\r
- IN OUT struct ibsp_socket_info *socket_info,\r
- IN int old_state )\r
+ IN OUT struct ibsp_socket_info *socket_info )\r
{\r
+ enum ibsp_socket_state old_state;\r
+\r
CL_ENTER( IBSP_DBG_EP, gdbg_lvl );\r
\r
+ cl_spinlock_acquire( &socket_info->mutex );\r
+ old_state = socket_info->socket_state;\r
+ IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSED );\r
+ cl_spinlock_release( &socket_info->mutex );\r
+\r
if( socket_info->duplicate.mmap_handle )\r
{\r
CloseHandle( socket_info->duplicate.mmap_handle );\r
socket_info->duplicate.mmap_handle = NULL;\r
}\r
\r
- if( socket_info->info.listen.handle )\r
+ if( socket_info->listen.handle )\r
{\r
/* Stop listening and reject queued connections. */\r
ib_listen_cancel( socket_info );\r
\r
switch( old_state )\r
{\r
- case IBSP_CLOSING:\r
- /* This function has already been called. Should not happen. */\r
- CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
- ("shutdown_and_destroy_socket_info already in closing socket_state\n") );\r
- return;\r
- break;\r
-\r
case IBSP_CREATE:\r
/* Nothing to do. */\r
break;\r
break;\r
}\r
\r
+ /* Flush all completions. */\r
+ if( socket_info->dup_cnt )\r
+ ibsp_dup_overlap_abort( socket_info );\r
+\r
+ while( socket_info->send_cnt || socket_info->recv_cnt )\r
+ ib_cq_comp( socket_info->cq_tinfo );\r
+\r
+ ib_deregister_all_mr( &socket_info->buf_mem_list );\r
+\r
+ ib_destroy_socket( socket_info );\r
+\r
+ /* Release the initial reference and clean up. */\r
+ deref_socket_info( socket_info );\r
+\r
CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
}\r
\r
return INVALID_SOCKET;\r
}\r
\r
- if( cl_qlist_count( &socket_info->info.listen.list ) == 0 )\r
+ if( cl_qlist_count( &socket_info->listen.list ) == 0 )\r
{\r
cl_spinlock_release( &socket_info->mutex );\r
\r
CL_TRACE( IBSP_DBG_CONN, gdbg_lvl,\r
("IBSPAccept: Found pending connection on this socket\n") );\r
\r
- incoming = PARENT_STRUCT(cl_qlist_head( &socket_info->info.listen.list ),\r
+ incoming = PARENT_STRUCT(cl_qlist_head( &socket_info->listen.list ),\r
struct listen_incoming, item);\r
port = socket_info->port;\r
\r
\r
reject:\r
/* The request is invalid. Remove it from the list and reject it. */\r
- cl_qlist_remove_item( &socket_info->info.listen.list, &incoming->item );\r
+ cl_qlist_remove_item( &socket_info->listen.list, &incoming->item );\r
cl_spinlock_release( &socket_info->mutex );\r
\r
ib_reject( incoming->cm_req_received.h_cm_req, IB_REJ_INSUF_QP );\r
switch( ret )\r
{\r
case CF_REJECT:\r
- cl_qlist_remove_item( &socket_info->info.listen.list, &incoming->item );\r
+ cl_qlist_remove_item( &socket_info->listen.list, &incoming->item );\r
cl_spinlock_release( &socket_info->mutex );\r
\r
IBSP_TRACE1( IBSP_DBG_CONN,\r
/* Store the IP address and port number in the socket context */\r
new_socket_info->local_addr = incoming->params.dest;\r
\r
- cl_qlist_remove_item( &socket_info->info.listen.list, &incoming->item );\r
+ cl_qlist_remove_item( &socket_info->listen.list, &incoming->item );\r
/* Signal the event again if there are more connection requests. */\r
- if( cl_qlist_count( &socket_info->info.listen.list ) )\r
+ if( cl_qlist_count( &socket_info->listen.list ) )\r
ibsp_post_select_event( socket_info, FD_ACCEPT, 0 );\r
\r
cl_spinlock_release( &socket_info->mutex );\r
new_socket_info->peer_addr = incoming->params.source;\r
new_socket_info->switch_socket = new_socket;\r
\r
- new_socket_info->info.accept.event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
+ new_socket_info->h_event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
\r
cl_spinlock_acquire( &new_socket_info->mutex );\r
/* Update the state of the socket context */\r
STAT_DEC( wpusocket_num );\r
}\r
\r
- CloseHandle( new_socket_info->info.accept.event );\r
+ CloseHandle( new_socket_info->h_event );\r
\r
ib_destroy_socket( new_socket_info );\r
\r
cl_spinlock_release( &new_socket_info->mutex );\r
HeapFree( g_ibsp.heap, 0, incoming );\r
\r
- if( WaitForSingleObject( new_socket_info->info.accept.event, INFINITE ) == WAIT_OBJECT_0 )\r
+ if( WaitForSingleObject( new_socket_info->h_event, INFINITE ) == WAIT_OBJECT_0 )\r
{\r
- CloseHandle( new_socket_info->info.accept.event );\r
+ CloseHandle( new_socket_info->h_event );\r
+ new_socket_info->h_event = NULL;\r
\r
cl_spinlock_acquire( &new_socket_info->mutex );\r
\r
}\r
else\r
{\r
- CloseHandle( new_socket_info->info.accept.event );\r
+ CloseHandle( new_socket_info->h_event );\r
+ new_socket_info->h_event = NULL;\r
\r
CL_EXIT_ERROR( IBSP_DBG_CONN, gdbg_lvl, ("wait for ib_accept failed\n") );\r
\r
LPINT lpErrno )\r
{\r
struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s;\r
- int old_state;\r
\r
CL_ENTER( IBSP_DBG_CONN, gdbg_lvl );\r
\r
cl_atomic_inc( &g_ibsp.CloseSocket_count );\r
#endif\r
\r
- cl_spinlock_acquire( &socket_info->mutex );\r
-\r
- old_state = socket_info->socket_state;\r
- IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSING );\r
-\r
- cl_spinlock_release( &socket_info->mutex );\r
-\r
- shutdown_and_destroy_socket_info( socket_info, old_state );\r
-\r
- cl_spinlock_acquire( &socket_info->mutex );\r
- IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSED );\r
- cl_spinlock_release( &socket_info->mutex );\r
-\r
- /* Take off socket_info_list and put on closed_socket_info_list */\r
cl_spinlock_acquire( &g_ibsp.socket_info_mutex );\r
cl_qlist_remove_item( &g_ibsp.socket_info_list, &socket_info->item );\r
cl_spinlock_release( &g_ibsp.socket_info_mutex );\r
\r
- cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
- cl_qlist_insert_tail( &g_ibsp.closed_socket_info_list, &socket_info->item );\r
- cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
+ shutdown_and_destroy_socket_info( socket_info );\r
\r
- /* Notify ib_cleanup_thread() to free this */\r
- SetEvent( g_ibsp.ib_cleanup_event );\r
\r
CL_EXIT( IBSP_DBG_CONN, gdbg_lvl );\r
\r
case IBSP_BIND:\r
\r
/* Store the backlog value in the context */\r
- socket_info->info.listen.backlog = backlog;\r
+ socket_info->listen.backlog = backlog;\r
IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_LISTEN );\r
\r
- socket_info->info.listen.listen_req_param.dwProcessId = 0;\r
- socket_info->info.listen.listen_req_param.identifier = 0;\r
+ socket_info->listen.listen_req_param.dwProcessId = 0;\r
+ socket_info->listen.listen_req_param.identifier = 0;\r
\r
ret = ib_listen( socket_info, lpErrno );\r
if( ret )\r
free_socket_info(\r
struct ibsp_socket_info *socket_info );\r
\r
+void\r
+deref_socket_info(\r
+ IN struct ibsp_socket_info *p_socket );\r
+\r
/* protos from extension.c */\r
HANDLE WSPAPI\r
IBSPRegisterMemory(\r
\r
void\r
shutdown_and_destroy_socket_info(\r
- IN OUT struct ibsp_socket_info *socket_info,\r
- IN int old_state );\r
+ IN OUT struct ibsp_socket_info *socket_info );\r
\r
void\r
destroy_qp_and_release_ressources(\r
IBSP_DUPLICATING_NEW, /* duplicating socket on the new controlling process */\r
IBSP_DUPLICATING_REMOTE, /* duplicating socket on the remote side */\r
IBSP_DISCONNECTED,\r
- IBSP_CLOSING,\r
IBSP_CLOSED,\r
IBSP_NUM_STATES\r
};\r
\r
enum ibsp_socket_state socket_state; /* represents current socket state */\r
\r
- union\r
+ struct\r
{\r
- struct\r
- {\r
- /* Listening socket */\r
- unsigned int backlog; /* Maximum number of pending connections */\r
- cl_qlist_t list; /* list of pending connections */\r
- ib_listen_handle_t handle;\r
- struct listen_req_param listen_req_param;\r
- } listen;\r
-\r
- struct\r
- {\r
- /* Accepting socket */\r
- /* TODO: ib_cm_rep will be blocking with some flags. \r
- * Use that instead of this event. */\r
- HANDLE event;\r
- } accept;\r
-\r
- struct\r
- {\r
- HANDLE event;\r
- } connect;\r
-\r
- struct\r
- {\r
- HANDLE event;\r
- } disconnect;\r
-\r
- } info;\r
+ /* Listening socket */\r
+ unsigned int backlog; /* Maximum number of pending connections */\r
+ cl_qlist_t list; /* list of pending connections */\r
+ ib_listen_handle_t handle;\r
+ struct listen_req_param listen_req_param;\r
+ } listen;\r
+\r
+ /* Event for blocking accept, connect, and disconnect */\r
+ HANDLE h_event;\r
\r
/* Variables associated with IBSPSelectEvent */\r
WSAEVENT event_select; /* Handle to Event Object */\r
ib_send_opt_t send_opt;\r
struct _wr send_wr[QP_ATTRIB_SQ_DEPTH];\r
uint8_t send_idx;\r
- atomic32_t send_cnt;\r
+ atomic32_t send_cnt; /* Used to limit access to send_wr array. */\r
\r
/* Receive request processing. */\r
cl_spinlock_t recv_lock;\r
struct _recv_wr recv_wr[QP_ATTRIB_RQ_DEPTH];\r
uint8_t recv_idx;\r
- atomic32_t recv_cnt;\r
+ atomic32_t recv_cnt; /* Used to limit access to recv_wr array. */\r
+\r
+ /*\r
+ * Used to stall destruction of switch socket until all completion\r
+ * upcalls have unwound.\r
+ */\r
+ atomic32_t ref_cnt;\r
\r
#ifdef _DEBUG_\r
atomic32_t send_comp;\r
return;\r
}\r
\r
- /* \r
- * Once in IBSP_CLOSING state, the only allowed new state\r
- * is IBSP_CLOSING.\r
- */\r
- if( (old_state == IBSP_CLOSING) && (new_state != IBSP_CLOSED) )\r
- {\r
- fzprint(("%s():0x%x:0x%x: socket=0x%p cannot change from %s to %s, called by %s():%d\n", __FUNCTION__, GetCurrentProcessId(), GetCurrentThreadId(), s, IBSP_SOCKET_STATE_STR(old_state), IBSP_SOCKET_STATE_STR(new_state), calling_func, line));\r
- return;\r
- }\r
-\r
fzprint(("%s():0x%x:0x%x: socket=0x%p %s to %s, called by %s():%d\n", __FUNCTION__,\r
GetCurrentProcessId(),\r
GetCurrentThreadId(), s,\r
cl_rbmap_t conn_map; /* rb tree of all connections to ensure unique 4-tuple */\r
cl_spinlock_t socket_info_mutex;\r
\r
- cl_qlist_t closed_socket_info_list; /* List of sockets in IBSP_CLOSED state */\r
- cl_spinlock_t closed_socket_info_mutex;\r
-\r
WSAPROTOCOL_INFOW protocol_info;\r
\r
/* Infiniband */\r
cl_fmap_t ip_map; /* list of all IP addresses supported by all the ports. */\r
cl_spinlock_t ip_mutex;\r
\r
- /* IB resource cleanup thread */\r
- HANDLE ib_cleanup_thread;\r
- HANDLE ib_cleanup_event;\r
- BOOL ib_cleanup_thread_exit_wanted;\r
-\r
#ifdef _DEBUG_\r
/* Statistics */\r
atomic32_t qp_num;\r
"IBSP_DUPLICATING_NEW",\r
"IBSP_DUPLICATING_REMOTE",\r
"IBSP_DISCONNECTED",\r
- "IBSP_CLOSING",\r
"IBSP_CLOSED"\r
};\r
\r
cl_rbmap_init( &g_ibsp.conn_map );\r
cl_spinlock_init( &g_ibsp.socket_info_mutex );\r
\r
- cl_qlist_init( &g_ibsp.closed_socket_info_list );\r
- cl_spinlock_init( &g_ibsp.closed_socket_info_mutex );\r
-\r
return 0;\r
}\r
\r
g_ibsp.heap = NULL;\r
\r
cl_spinlock_destroy( &g_ibsp.socket_info_mutex );\r
- cl_spinlock_destroy( &g_ibsp.closed_socket_info_mutex );\r
cl_spinlock_destroy( &g_ibsp.hca_mutex );\r
cl_spinlock_destroy( &g_ibsp.ip_mutex );\r
cl_spinlock_destroy( &g_ibsp.mutex );\r
cl_qlist_init( &socket_info->buf_mem_list.list );\r
cl_spinlock_init( &socket_info->buf_mem_list.mutex );\r
\r
- cl_qlist_init( &socket_info->info.listen.list );\r
+ cl_qlist_init( &socket_info->listen.list );\r
\r
cl_spinlock_init( &socket_info->send_lock );\r
cl_spinlock_init( &socket_info->recv_lock );\r
#endif\r
\r
socket_info->switch_socket = INVALID_SOCKET;\r
+ /*\r
+ * Preset to 1, IBSPCloseSocket will decrement it, and switch socket\r
+ * will be freed once it goes to zero after an upcall.\r
+ */\r
+ socket_info->ref_cnt = 1;\r
\r
IBSP_TRACE( IBSP_DBG_SI, ("socket_info (%p)\n", socket_info) );\r
\r
\r
IBSP_EXIT( IBSP_DBG_SI );\r
}\r
+\r
+\r
+void\r
+deref_socket_info(\r
+ IN struct ibsp_socket_info *p_socket )\r
+{\r
+ int ret, error;\r
+\r
+ if( !cl_atomic_dec( &p_socket->ref_cnt ) )\r
+ {\r
+ if( p_socket->switch_socket != INVALID_SOCKET )\r
+ {\r
+ /* ref_cnt hit zero - destroy the switch socket. */\r
+ IBSP_TRACE1( IBSP_DBG_SI,\r
+ ("socket=0x%p calling lpWPUCloseSocketHandle=0x%p\n",\r
+ p_socket, p_socket->switch_socket) );\r
+\r
+ ret = g_ibsp.up_call_table.lpWPUCloseSocketHandle(\r
+ p_socket->switch_socket, &error );\r
+ if( ret == SOCKET_ERROR )\r
+ {\r
+ IBSP_ERROR( ("WPUCloseSocketHandle failed: %d\n", error) );\r
+ }\r
+ else\r
+ {\r
+ STAT_DEC( wpusocket_num );\r
+ }\r
+\r
+ p_socket->switch_socket = INVALID_SOCKET;\r
+ }\r
+\r
+ /*\r
+ * This only works because the socket doesn't have its own CQ. If it\r
+ * did CQ destruction would have to be async since IBSPCloseSocket can\r
+ * be invoked from the CQ callback thread context, and freeing the\r
+ * socket structure would need to be delayed until the CQ's destroy\r
+ * callback.\r
+ */\r
+ free_socket_info( p_socket );\r
+ }\r
+}\r
projects / ~shefty / rdma-win.git / commitdiff