From c8f6270b27c46a6cd6c620fedac7ee0c0d30c4f3 Mon Sep 17 00:00:00 2001 From: sleybo Date: Mon, 19 Feb 2007 15:39:02 +0000 Subject: [PATCH] [IPOIB] bug fix: destroy wait for multicast leave request to finish git-svn-id: svn://openib.tc.cornell.edu/gen1@595 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- trunk/ulp/ipoib/kernel/ipoib_adapter.c | 41 ++++++++--- trunk/ulp/ipoib/kernel/ipoib_adapter.h | 1 + trunk/ulp/ipoib/kernel/ipoib_debug.h | 1 + trunk/ulp/ipoib/kernel/ipoib_endpoint.c | 18 +++-- trunk/ulp/ipoib/kernel/ipoib_port.c | 98 +++++++++++++++++++++++-- trunk/ulp/ipoib/kernel/ipoib_port.h | 10 +++ 6 files changed, 145 insertions(+), 24 deletions(-) diff --git a/trunk/ulp/ipoib/kernel/ipoib_adapter.c b/trunk/ulp/ipoib/kernel/ipoib_adapter.c index 0434814f..27e0cb18 100644 --- a/trunk/ulp/ipoib/kernel/ipoib_adapter.c +++ b/trunk/ulp/ipoib/kernel/ipoib_adapter.c @@ -92,16 +92,16 @@ __ipoib_pnp_dereg( IN void* context ); +static void +__ipoib_adapter_reset( + IN void* context); + + static ib_api_status_t __ipoib_pnp_cb( IN ib_pnp_rec_t *p_pnp_rec ); -static void -ipoib_destroy_port( - IN ipoib_adapter_t* const p_adapter ); - - void ipoib_join_mcast( IN ipoib_adapter_t* const p_adapter ); @@ -274,6 +274,9 @@ adapter_construct( cl_spinlock_construct( &p_adapter->recv_stat_lock ); cl_qpool_construct( &p_adapter->item_pool ); KeInitializeMutex( &p_adapter->mutex, 0 ); + + cl_thread_construct(&p_adapter->destroy_thread); + cl_vector_construct( &p_adapter->ip_vector ); cl_perf_construct( &p_adapter->perf ); @@ -337,6 +340,7 @@ adapter_init( return IB_ERROR; } + /* Validate the port GUID and generate the MAC address. */ status = ipoib_mac_from_guid( p_adapter->guids.port_guid, &p_adapter->mac ); @@ -348,6 +352,8 @@ adapter_init( return status; } + + /* Open AL. */ status = p_adapter->p_ifc->open_al( &p_adapter->h_al ); if( status != IB_SUCCESS ) @@ -377,7 +383,7 @@ __ipoib_pnp_reg( CL_ASSERT( !p_adapter->registering ); p_adapter->registering = TRUE; - + /* Register for PNP events. */ cl_memclr( &pnp_req, sizeof(pnp_req) ); pnp_req.pnp_class = IB_PNP_PORT | flags; @@ -812,14 +818,31 @@ __ipoib_pnp_dereg( IN void* context ) { ipoib_adapter_t* p_adapter; - ipoib_port_t* p_port; - ib_api_status_t status; - ib_pnp_event_t state; IPOIB_ENTER( IPOIB_DBG_INIT ); p_adapter = PARENT_STRUCT( context, ipoib_adapter_t, obj ); + cl_thread_init(&p_adapter->destroy_thread, __ipoib_adapter_reset, (void*)p_adapter, "destroy_thread"); + + IPOIB_ENTER( IPOIB_DBG_INIT ); + +} + +static void +__ipoib_adapter_reset( + IN void* context) +{ + + ipoib_adapter_t *p_adapter; + ipoib_port_t *p_port; + ib_api_status_t status; + ib_pnp_event_t state; + + IPOIB_ENTER( IPOIB_DBG_INIT ); + + p_adapter = (ipoib_adapter_t*)context; + /* Synchronize with destruction */ KeWaitForMutexObject( &p_adapter->mutex, Executive, KernelMode, FALSE, NULL ); diff --git a/trunk/ulp/ipoib/kernel/ipoib_adapter.h b/trunk/ulp/ipoib/kernel/ipoib_adapter.h index 34cc7764..58de4eff 100644 --- a/trunk/ulp/ipoib/kernel/ipoib_adapter.h +++ b/trunk/ulp/ipoib/kernel/ipoib_adapter.h @@ -177,6 +177,7 @@ typedef struct _ipoib_adapter KMUTEX mutex; + cl_thread_t destroy_thread; cl_vector_t ip_vector; cl_perf_t perf; diff --git a/trunk/ulp/ipoib/kernel/ipoib_debug.h b/trunk/ulp/ipoib/kernel/ipoib_debug.h index 7c9852a6..6a5e3c7a 100644 --- a/trunk/ulp/ipoib/kernel/ipoib_debug.h +++ b/trunk/ulp/ipoib/kernel/ipoib_debug.h @@ -260,6 +260,7 @@ enum ref_cnt_buckets ref_get_bcast, ref_bcast, /* join and create, used as base only */ ref_join_mcast, + ref_leave_mcast, ref_endpt_track, /* used when endpt is in port's child list. */ ref_array_size, /* Used to size the array of ref buckets. */ diff --git a/trunk/ulp/ipoib/kernel/ipoib_endpoint.c b/trunk/ulp/ipoib/kernel/ipoib_endpoint.c index 55fda056..d727b02e 100644 --- a/trunk/ulp/ipoib/kernel/ipoib_endpoint.c +++ b/trunk/ulp/ipoib/kernel/ipoib_endpoint.c @@ -207,6 +207,16 @@ __endpt_destroying( p_port->p_adapter->h_al, p_endpt->h_query ); p_endpt->h_query = NULL; } + + /* Leave the multicast group if it exists. */ + if( p_endpt->h_mcast ) + { + IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT, + ("Leaving MCast group\n") ); + ipoib_port_ref(p_port, ref_leave_mcast); + p_port->p_adapter->p_ifc->leave_mcast( p_endpt->h_mcast, ipoib_leave_mcast_cb ); + } + cl_obj_unlock( p_obj ); IPOIB_EXIT( IPOIB_DBG_ENDPT ); @@ -225,14 +235,6 @@ __endpt_cleanup( p_endpt = PARENT_STRUCT( p_obj, ipoib_endpt_t, obj ); p_port = __endpt_parent( p_endpt ); - /* Leave the multicast group if it exists. */ - if( p_endpt->h_mcast ) - { - IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT, - ("Leaving MCast group\n") ); - p_port->p_adapter->p_ifc->leave_mcast( p_endpt->h_mcast, NULL ); - } - /* Destroy the AV if it exists. */ if( p_endpt->h_av ) p_port->p_adapter->p_ifc->destroy_av( p_endpt->h_av ); diff --git a/trunk/ulp/ipoib/kernel/ipoib_port.c b/trunk/ulp/ipoib/kernel/ipoib_port.c index 70e80f26..29b299a7 100644 --- a/trunk/ulp/ipoib/kernel/ipoib_port.c +++ b/trunk/ulp/ipoib/kernel/ipoib_port.c @@ -449,6 +449,10 @@ static void __mcast_cb( IN ib_mcast_rec_t *p_mcast_rec ); +void +__leave_error_mcast_cb( + IN void *context ); + static intn_t __gid_cmp( @@ -573,7 +577,8 @@ __port_construct( __endpt_mgr_construct( p_port ); KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE ); - + KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE ); + IPOIB_EXIT( IPOIB_DBG_INIT ); } @@ -4047,8 +4052,10 @@ __endpt_mgr_reset_all( IN ipoib_port_t* const p_port ) { cl_map_item_t *p_item; - ipoib_endpt_t *p_endpt; - cl_qlist_t mc_list; + ipoib_endpt_t *p_endpt; + cl_qlist_t mc_list; + uint32_t local_exist = 0; + IPOIB_ENTER( IPOIB_DBG_ENDPT ); @@ -4070,6 +4077,7 @@ __endpt_mgr_reset_all( cl_qlist_insert_head( &mc_list, &p_port->p_local_endpt->mac_item.pool_item.list_item ); + local_exist = 1; p_port->p_local_endpt = NULL; } @@ -4108,8 +4116,22 @@ __endpt_mgr_reset_all( } } + cl_obj_unlock( &p_port->obj ); + + if(cl_qlist_count( &mc_list ) - local_exist) + { + p_port->mcast_cnt = (uint32_t)cl_qlist_count( &mc_list ) - local_exist; + } + else + { + p_port->mcast_cnt = 0; + KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE ); + } + + IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt - local_exist)); + /* Destroy all multicast endpoints now that we have released the lock. */ while( cl_qlist_count( &mc_list ) ) { @@ -5109,9 +5131,14 @@ ipoib_port_down( return; } + KeResetEvent(&p_port->leave_mcast_event); + /* Reset all endpoints so we don't flush our ARP cache. */ __endpt_mgr_reset_all( p_port ); + KeWaitForSingleObject( + &p_port->leave_mcast_event, Executive, KernelMode, FALSE, NULL ); + cl_obj_lock( &p_port->p_adapter->obj ); ipoib_dereg_addrs( p_port->p_adapter ); cl_obj_unlock( &p_port->p_adapter->obj ); @@ -5136,8 +5163,11 @@ __bcast_cb( { cl_obj_unlock( &p_port->obj ); if( p_mcast_rec->status == IB_SUCCESS ) - p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL ); + { + ipoib_port_ref(p_port, ref_leave_mcast); + p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb ); + } KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE ); ipoib_port_deref( p_port, ref_bcast_inv_state ); IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT, @@ -5197,7 +5227,8 @@ __bcast_cb( IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR, ("__endpt_mgr_add_bcast returned %s\n", p_port->p_adapter->p_ifc->get_err_str( status )) ); - status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL ); + ipoib_port_ref(p_port, ref_leave_mcast); + status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb ); CL_ASSERT( status == IB_SUCCESS ); goto err; } @@ -5453,8 +5484,11 @@ __mcast_cb( { cl_obj_unlock( &p_port->obj ); if( p_mcast_rec->status == IB_SUCCESS ) - p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL ); + { + ipoib_port_ref(p_port, ref_leave_mcast); + p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb ); + } ipoib_port_deref( p_port, ref_mcast_inv_state ); IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT, ("Invalid state - Aborting.\n") ); @@ -5487,7 +5521,9 @@ __mcast_cb( cl_obj_unlock( &p_port->obj ); IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR, ("Failed to find endpoint for update.\n") ); - p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL ); + + ipoib_port_ref(p_port, ref_leave_mcast); + p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb ); ipoib_port_deref( p_port, ref_mcast_no_endpt ); IPOIB_EXIT( IPOIB_DBG_MCAST ); return; @@ -5531,3 +5567,51 @@ __mcast_cb( IPOIB_EXIT( IPOIB_DBG_MCAST ); } + + +void +ipoib_leave_mcast_cb( + IN void *context ) +{ + ipoib_port_t *p_port; + + IPOIB_ENTER( IPOIB_DBG_MCAST ); + + p_port = (ipoib_port_t* __ptr64)context; + + IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt)); + + ipoib_port_deref( p_port, ref_leave_mcast); + cl_atomic_dec( &p_port->mcast_cnt); + + if(0 == p_port->mcast_cnt) + { + KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE ); + } + + IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST, + ("Leav mcast callback deref ipoib_port \n") ); + + IPOIB_EXIT( IPOIB_DBG_MCAST ); +} + + + +void +__leave_error_mcast_cb( + IN void *context ) +{ + ipoib_port_t *p_port; + + IPOIB_ENTER( IPOIB_DBG_MCAST ); + + p_port = (ipoib_port_t* __ptr64)context; + + ipoib_port_deref( p_port, ref_leave_mcast); + IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST, + ("Leav mcast callback deref ipoib_port \n") ); + + IPOIB_EXIT( IPOIB_DBG_MCAST ); +} + + diff --git a/trunk/ulp/ipoib/kernel/ipoib_port.h b/trunk/ulp/ipoib/kernel/ipoib_port.h index 046d1500..443ec61a 100644 --- a/trunk/ulp/ipoib/kernel/ipoib_port.h +++ b/trunk/ulp/ipoib/kernel/ipoib_port.h @@ -484,6 +484,10 @@ typedef struct _ipoib_port uint8_t port_num; KEVENT sa_event; + + atomic32_t mcast_cnt; + KEVENT leave_mcast_event; + ipoib_ib_mgr_t ib_mgr; ipoib_buf_mgr_t buf_mgr; @@ -568,6 +572,12 @@ ipoib_port_join_mcast( IN const mac_addr_t mac, IN const uint8_t state ); + +void +ipoib_leave_mcast_cb( + IN void *context ); + + void ipoib_port_remove_endpt( IN ipoib_port_t* const p_port, -- 2.41.0