From 68fd91f3d848c80b4b67d8c6b41219b74f686da3 Mon Sep 17 00:00:00 2001 From: ftillier Date: Tue, 10 Jan 2006 19:49:14 +0000 Subject: [PATCH] [WSD] Fix socket duplication to allow the parent application to close its socket before the child has completed duplication. git-svn-id: svn://openib.tc.cornell.edu/gen1@213 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/ulp/wsd/user/ibsp_duplicate.c | 250 +++++++++++++--------------- trunk/ulp/wsd/user/ibsp_iblow.c | 6 - trunk/ulp/wsd/user/ibspdebug.h | 1 + trunk/ulp/wsd/user/ibspdll.c | 62 +------ trunk/ulp/wsd/user/ibspproto.h | 12 +- trunk/ulp/wsd/user/ibspstruct.h | 9 +- 6 files changed, 131 insertions(+), 209 deletions(-) diff --git a/trunk/ulp/wsd/user/ibsp_duplicate.c b/trunk/ulp/wsd/user/ibsp_duplicate.c index 46fd7247..1838054d 100644 --- a/trunk/ulp/wsd/user/ibsp_duplicate.c +++ b/trunk/ulp/wsd/user/ibsp_duplicate.c @@ -41,9 +41,15 @@ static void create_name( OUT char *fname, IN const DWORD dwProcessId, - IN const DWORD identifier ) + IN const GUID *p_guid ) { - sprintf( fname, "SilverStorm-WSD-%08lx-%08lx", dwProcessId, identifier ); + sprintf( fname, "Global\\OpenIB-WSD-%08lx-" + "%08lx-%04hx-%04hx-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x", + dwProcessId, p_guid->Data1, p_guid->Data2, p_guid->Data3, + (int)p_guid->Data4[0], (int)p_guid->Data4[1], + (int)p_guid->Data4[2], (int)p_guid->Data4[3], + (int)p_guid->Data4[4], (int)p_guid->Data4[5], + (int)p_guid->Data4[6], (int)p_guid->Data4[7] ); } @@ -53,53 +59,19 @@ create_name( int setup_duplicate_socket( IN struct ibsp_socket_info *socket_info, - IN DWORD identifier ) + IN HANDLE h_dup_info ) { - char fname[100]; - HANDLE h = NULL; int ret, err; struct ibsp_duplicate_info *dup_info; ib_net64_t dest_port_guid; ib_path_rec_t path_rec; - IBSP_ENTER( IBSP_DBG_CONN ); + IBSP_ENTER( IBSP_DBG_DUP ); CL_ASSERT( socket_info->socket_state == IBSP_CREATE ); - fzprint(("%s():%d:0x%x:0x%x: socket=0x%p dwProcessId=0x%x\n", __FUNCTION__, - __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info)); - - - /* Find the info created and shared by the previous controlling socket. */ - create_name( fname, GetCurrentProcessId(), identifier ); - - h = CreateFileMapping( INVALID_HANDLE_VALUE, // use paging file - NULL, // default security attributes - PAGE_READWRITE, // read access - 0, // size: high 32-bits - sizeof(struct ibsp_duplicate_info), // size: low 32-bits - fname ); // name of map object - if( h == NULL ) - { - IBSP_ERROR( ("CreateFileMapping failed with %d\n", GetLastError()) ); - ret = WSAENETDOWN; - goto err1; - } - - /* Make sure this file already existed. */ - if( GetLastError() != ERROR_ALREADY_EXISTS ) - { - IBSP_ERROR( ("not mapping for socket duplicate info\n") ); - ret = WSAENETDOWN; - goto err1; - } - /* Get a pointer to the file-mapped shared memory. */ - dup_info = MapViewOfFile( h, // object to map view of - FILE_MAP_WRITE, // read/write access - 0, // high offset: map from - 0, // low offset: beginning - 0 ); // default: map entire file + dup_info = MapViewOfFile( h_dup_info, FILE_MAP_READ, 0, 0, 0 ); if( dup_info == NULL ) { IBSP_ERROR( ("MapViewOfFile failed with %d\n", GetLastError()) ); @@ -111,10 +83,8 @@ setup_duplicate_socket( socket_info->local_addr = dup_info->local_addr; socket_info->socket_options = dup_info->socket_options; socket_info->duplicate.dwProcessId = dup_info->dwProcessId; - socket_info->duplicate.identifier = identifier; + socket_info->duplicate.identifier = dup_info->identifier; - /* Get the port info */ - /* Find the destination IP address */ socket_info->port = get_port_from_ip_address( dup_info->local_addr.sin_addr ); if( socket_info->port == NULL ) { @@ -192,143 +162,153 @@ err2: } err1: - if( h ) - CloseHandle( h ); - if( socket_info->h_event ) { CloseHandle( socket_info->h_event ); socket_info->h_event = NULL; } - IBSP_EXIT( IBSP_DBG_CONN ); + CloseHandle( h_dup_info ); + + IBSP_EXIT( IBSP_DBG_DUP ); return ret; } -/* - * This function is called by the previous controlling process. - * Called with the socket_info->mutex held. - */ -int -prepare_duplicate_socket( - IN struct ibsp_socket_info *socket_info, - IN DWORD dwProcessId ) +/* Function: IBSPDuplicateSocket + + Description: + This function provides a WSAPROTOCOL_INFOW structure which can be passed + to another process to open a handle to the same socket. First we need + to translate the user socket into the provider socket and call the underlying + WSPDuplicateSocket. Note that the lpProtocolInfo structure passed into us + is an out parameter only! +*/ +int WSPAPI +IBSPDuplicateSocket( + SOCKET s, + DWORD dwProcessId, + LPWSAPROTOCOL_INFOW lpProtocolInfo, + LPINT lpErrno ) { + struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s; struct ibsp_duplicate_info *dup_info = NULL; - int i; - DWORD identifier = 0; + char fname[100]; + GUID guid; + HANDLE h_dup_info, h_target_process, h_target_dup_info; struct disconnect_reason reason; - IBSP_ENTER( IBSP_DBG_CONN ); + IBSP_ENTER( IBSP_DBG_DUP ); - fzprint(("%s():%d:0x%x:0x%x: socket=0x%p dwProcessId=0x%x \n", __FUNCTION__, - __LINE__, GetCurrentProcessId(), - GetCurrentThreadId(), socket_info, dwProcessId)); + IBSP_TRACE4( IBSP_DBG_DUP, + ("Duplicating socket=0x%p to dwProcessId=0x%x \n", + socket_info, dwProcessId) ); + + cl_spinlock_acquire( &socket_info->mutex ); + if( socket_info->socket_state != IBSP_CONNECTED ) + { + cl_spinlock_release( &socket_info->mutex ); + IBSP_TRACE_EXIT( IBSP_DBG_DUP, + ("Socket state not IBSP_CONNECTED, state=%s.\n", + IBSP_SOCKET_STATE_STR( socket_info->socket_state )) ); + *lpErrno = WSAENOTCONN; + return SOCKET_ERROR; + } - /* First, flush all the receive buffers. There should be no send/rdma buffers left. */ - CL_ASSERT( socket_info->socket_state == IBSP_CONNECTED ); IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_DUPLICATING_OLD ); /* We changed the state - remove from connection map. */ ibsp_conn_remove( socket_info ); - /* We might have a left-over from the previous duplicate. */ - if( socket_info->duplicate.mmap_handle != NULL ) + cl_spinlock_release( &socket_info->mutex ); + + /* Create a GUID to use as unique identifier for this duplication. */ + UuidCreate( &guid ); + create_name( fname, dwProcessId, &guid ); + + h_dup_info = CreateFileMapping( INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE, 0, sizeof(struct ibsp_duplicate_info), fname ); + if( !h_dup_info ) { - CloseHandle( socket_info->duplicate.mmap_handle ); - socket_info->duplicate.mmap_handle = NULL; + IBSP_ERROR_EXIT( + ("CreateFileMapping for %s failed with %d\n", + fname, GetLastError()) ); + *lpErrno = WSAENETDOWN; + return SOCKET_ERROR; + } + + /* Get a pointer to the file-mapped shared memory. */ + dup_info = MapViewOfFile( h_dup_info, FILE_MAP_WRITE, 0, 0, 0 ); + if( !dup_info ) + { + IBSP_ERROR_EXIT( + ("MapViewOfFile failed with %d\n", GetLastError()) ); + CloseHandle( h_dup_info ); + *lpErrno = WSAENETDOWN; + return SOCKET_ERROR; } /* - * Share this socket_info so the next-controlling process can get the necessary - * information to create the duplicated socket. Ensure a unique identifier. + * Store addressing information so that the duplicating + * process can reconnect. */ - for( i = 0; i < 50; i++ ) + dup_info->identifier = guid; + dup_info->socket_options = socket_info->socket_options; + dup_info->peer_addr = socket_info->peer_addr; + dup_info->local_addr = socket_info->local_addr; + dup_info->dwProcessId = dwProcessId; + + /* Release the reference on the underlying file */ + UnmapViewOfFile( dup_info ); + + /* Open the target process. */ + h_target_process = OpenProcess( PROCESS_DUP_HANDLE, FALSE, dwProcessId ); + if( !h_target_process ) { - char fname[100]; - GUID guid; - - /* Create a GUID and collapse it into 32 bits. */ - UuidCreate( &guid ); - identifier = guid.Data1 ^ (guid.Data2 << 16 | guid.Data3) ^ - (guid.Data4[7] << 24 | guid.Data4[6] << 16 | guid.Data4[5] << 8 | guid. - Data4[4]) ^ (guid.Data4[3] << 24 | guid.Data4[2] << 16 | guid. - Data4[1] << 8 | guid.Data4[0]); - - create_name( fname, dwProcessId, identifier ); - - socket_info->duplicate.mmap_handle = CreateFileMapping( INVALID_HANDLE_VALUE, // use paging file - NULL, // default security attributes - PAGE_READWRITE, // read access - 0, // size: high 32-bits - sizeof(struct ibsp_duplicate_info), // size: low 32-bits - fname ); // name of map object - - if( socket_info->duplicate.mmap_handle == NULL ) - { - IBSP_ERROR_EXIT( - ("CreateFileMapping failed with %d\n", GetLastError()) ); - return WSAENETDOWN; - } - - /* Make sure this identifier is unique for that process. */ - if( GetLastError() == ERROR_ALREADY_EXISTS ) - { - CloseHandle( socket_info->duplicate.mmap_handle ); - socket_info->duplicate.mmap_handle = NULL; - - /* Try again. */ - continue; - } - - /* Get a pointer to the file-mapped shared memory. */ - dup_info = MapViewOfFile( socket_info->duplicate.mmap_handle, // object to map view of - FILE_MAP_WRITE, // read/write access - 0, // high offset: map from - 0, // low offset: beginning - 0 ); // default: map entire file - if( dup_info == NULL ) - { - IBSP_ERROR_EXIT( - ("MapViewOfFile failed with %d\n", GetLastError()) ); - CloseHandle( socket_info->duplicate.mmap_handle ); - socket_info->duplicate.mmap_handle = NULL; - return WSAENETDOWN; - } + IBSP_ERROR_EXIT( + ("OpenProcess failed with %d\n", GetLastError()) ); + CloseHandle( h_dup_info ); + *lpErrno = WSAENETDOWN; + return SOCKET_ERROR; } - if( dup_info == NULL ) + if( !DuplicateHandle( GetCurrentProcess(), h_dup_info, + h_target_process, &h_target_dup_info, 0, TRUE, + DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS ) ) { - IBSP_ERROR_EXIT( ("failed to get a unique identifier\n") ); - CloseHandle( socket_info->duplicate.mmap_handle ); - socket_info->duplicate.mmap_handle = NULL; - return WSAENETDOWN; + IBSP_ERROR_EXIT( + ("DuplicateHandle failed with %d\n", GetLastError()) ); + CloseHandle( h_target_process ); + *lpErrno = WSAENETDOWN; + return SOCKET_ERROR; } - socket_info->duplicate.identifier = identifier; + CloseHandle( h_target_process ); + + CL_ASSERT( !((ULONG_PTR)h_target_dup_info >> 32) ); + lpProtocolInfo->dwProviderReserved = (DWORD)(ULONG_PTR)h_target_dup_info; + + socket_info->duplicate.identifier = guid; memset( &reason, 0, sizeof(reason) ); reason.type = DISC_DUPLICATING; - reason.duplicating.identifier = identifier; + reason.duplicating.identifier = guid; reason.duplicating.dwProcessId = dwProcessId; - cl_spinlock_release( &socket_info->mutex ); + /* + * Flush all the receive buffers. There should be no + * send/rdma buffers left. + */ ib_disconnect( socket_info, &reason ); wait_cq_drain( socket_info ); cl_spinlock_acquire( &socket_info->mutex ); ib_destroy_socket( socket_info ); - - /* Put enough info in dup_info so that the remote socket can recreate the connection. */ - dup_info->port_guid = socket_info->port->guid; - dup_info->socket_options = socket_info->socket_options; - dup_info->peer_addr = socket_info->peer_addr; - dup_info->local_addr = socket_info->local_addr; - dup_info->dwProcessId = dwProcessId; + cl_spinlock_release( &socket_info->mutex ); /* And that's it */ - IBSP_EXIT( IBSP_DBG_CONN ); + IBSP_EXIT( IBSP_DBG_DUP ); + *lpErrno = 0; return 0; } diff --git a/trunk/ulp/wsd/user/ibsp_iblow.c b/trunk/ulp/wsd/user/ibsp_iblow.c index 0122fbed..87c318e1 100644 --- a/trunk/ulp/wsd/user/ibsp_iblow.c +++ b/trunk/ulp/wsd/user/ibsp_iblow.c @@ -1098,12 +1098,6 @@ shutdown_and_destroy_socket_info( IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSED ); cl_spinlock_release( &socket_info->mutex ); - if( socket_info->duplicate.mmap_handle ) - { - CloseHandle( socket_info->duplicate.mmap_handle ); - socket_info->duplicate.mmap_handle = NULL; - } - if( socket_info->listen.handle ) { /* Stop listening and reject queued connections. */ diff --git a/trunk/ulp/wsd/user/ibspdebug.h b/trunk/ulp/wsd/user/ibspdebug.h index 3a9f84e7..5fed257b 100644 --- a/trunk/ulp/wsd/user/ibspdebug.h +++ b/trunk/ulp/wsd/user/ibspdebug.h @@ -48,6 +48,7 @@ extern uint32_t gdbg_lvl; #define IBSP_DBG_NEV 0x00000400 /* network events */ #define IBSP_DBG_HW 0x00000800 /* Hardware */ #define IBSP_DBG_IO 0x00001000 /* Overlapped I/O request */ +#define IBSP_DBG_DUP 0x00002000 /* Socket Duplication */ #define IBSP_DBG_LEVEL4 0x01000000 /* debug use */ #define IBSP_DBG_LEVEL3 0x02000000 /* debug use */ diff --git a/trunk/ulp/wsd/user/ibspdll.c b/trunk/ulp/wsd/user/ibspdll.c index 1c372a7c..177b7357 100644 --- a/trunk/ulp/wsd/user/ibspdll.c +++ b/trunk/ulp/wsd/user/ibspdll.c @@ -825,61 +825,6 @@ done: } -/* Function: IBSPDuplicateSocket - - Description: - This function provides a WSAPROTOCOL_INFOW structure which can be passed - to another process to open a handle to the same socket. First we need - to translate the user socket into the provider socket and call the underlying - WSPDuplicateSocket. Note that the lpProtocolInfo structure passed into us - is an out parameter only! -*/ -static int WSPAPI -IBSPDuplicateSocket( - SOCKET s, - DWORD dwProcessId, - LPWSAPROTOCOL_INFOW lpProtocolInfo, - LPINT lpErrno ) -{ - struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s; - int ret; - - IBSP_ENTER( IBSP_DBG_CONN ); - - fzprint(("%s():%d:0x%x:0x%x: socket=0x%p state=%s\n", __FUNCTION__, - __LINE__, GetCurrentProcessId(), - GetCurrentThreadId(), s, IBSP_SOCKET_STATE_STR( socket_info->socket_state ))); - - cl_spinlock_acquire( &socket_info->mutex ); - if( socket_info->socket_state != IBSP_CONNECTED ) - { - cl_spinlock_release( &socket_info->mutex ); - IBSP_TRACE_EXIT( IBSP_DBG_CONN, - ("Socket state not IBSP_CONNECTED, state=%s.\n", - IBSP_SOCKET_STATE_STR( socket_info->socket_state )) ); - *lpErrno = WSAENOTCONN; - return SOCKET_ERROR; - } - - ret = prepare_duplicate_socket( socket_info, dwProcessId ); - cl_spinlock_release( &socket_info->mutex ); - if( ret ) - { - IBSP_TRACE_EXIT( IBSP_DBG_CONN, - ("prepare_duplicate_socket failed with %d\n", ret) ); - *lpErrno = ret; - return SOCKET_ERROR; - } - else - { - IBSP_EXIT( IBSP_DBG_CONN ); - lpProtocolInfo->dwProviderReserved = socket_info->duplicate.identifier; - *lpErrno = 0; - return 0; - } -} - - /* Function: IBSPEnumNetworkEvents * * Description: @@ -1431,7 +1376,8 @@ IBSPListen( IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_LISTEN ); socket_info->listen.listen_req_param.dwProcessId = 0; - socket_info->listen.listen_req_param.identifier = 0; + cl_memclr( &socket_info->listen.listen_req_param.identifier, + sizeof(socket_info->listen.listen_req_param.identifier) ); ret = ib_listen( socket_info ); if( ret ) @@ -2089,8 +2035,8 @@ IBSPSocket( if( lpProtocolInfo->dwProviderReserved != 0 ) { /* This is a duplicate socket. */ - *lpErrno = setup_duplicate_socket( - socket_info, lpProtocolInfo->dwProviderReserved ); + *lpErrno = setup_duplicate_socket( socket_info, + (HANDLE)(ULONG_PTR)lpProtocolInfo->dwProviderReserved ); if( *lpErrno ) { deref_socket_info( socket_info ); diff --git a/trunk/ulp/wsd/user/ibspproto.h b/trunk/ulp/wsd/user/ibspproto.h index 4f866f82..98ad7eaa 100644 --- a/trunk/ulp/wsd/user/ibspproto.h +++ b/trunk/ulp/wsd/user/ibspproto.h @@ -227,12 +227,14 @@ pnp_ca_remove( int setup_duplicate_socket( IN struct ibsp_socket_info *socket_info, - IN DWORD identifier ); + IN HANDLE h_dup_info ); -int -prepare_duplicate_socket( - IN struct ibsp_socket_info *socket_info, - IN DWORD dwProcessId ); +int WSPAPI +IBSPDuplicateSocket( + SOCKET s, + DWORD dwProcessId, + LPWSAPROTOCOL_INFOW lpProtocolInfo, + LPINT lpErrno ); /* ibsp_mem.c */ diff --git a/trunk/ulp/wsd/user/ibspstruct.h b/trunk/ulp/wsd/user/ibspstruct.h index fe04c402..4d152f24 100644 --- a/trunk/ulp/wsd/user/ibspstruct.h +++ b/trunk/ulp/wsd/user/ibspstruct.h @@ -73,7 +73,7 @@ struct ibsp_socket_options struct listen_req_param { DWORD dwProcessId; - DWORD identifier; + GUID identifier; }; /* Parameters given to establish a connection */ @@ -125,7 +125,7 @@ struct mr_list /* Information necessary to duplicate sockets */ struct ibsp_duplicate_info { - ib_net64_t port_guid; + GUID identifier; struct ibsp_socket_options socket_options; struct sockaddr_in local_addr; struct sockaddr_in peer_addr; @@ -144,7 +144,7 @@ struct disconnect_reason struct _disconnect_reason_dup { - DWORD identifier; + GUID identifier; DWORD dwProcessId; } duplicating; @@ -316,8 +316,7 @@ struct ibsp_socket_info /* Stuff for socket duplication */ struct { - HANDLE mmap_handle; - DWORD identifier; /* Unique identifier */ + GUID identifier; /* Unique identifier */ DWORD dwProcessId; } duplicate; -- 2.41.0