[IPOIB] bug fix: destroy wait for multicast leave request to finish

git-svn-id: svn://openib.tc.cornell.edu/gen1@595 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/ulp/ipoib/kernel/ipoib_adapter.c b/trunk/ulp/ipoib/kernel/ipoib_adapter.c
index 0434814fb47a46d6e88d6152219103d06dd15c4a..27e0cb181f56f4e8a9613087dca22ba50af2980d 100644 (file)
--- a/trunk/ulp/ipoib/kernel/ipoib_adapter.c
+++ b/trunk/ulp/ipoib/kernel/ipoib_adapter.c
@@ -92,16 +92,16 @@ __ipoib_pnp_dereg(
        IN                              void*                                           context );\r
 \r
 \r
+static void\r
+__ipoib_adapter_reset(\r
+       IN                              void*   context);\r
+\r
+\r
 static ib_api_status_t\r
 __ipoib_pnp_cb(\r
        IN                              ib_pnp_rec_t                            *p_pnp_rec );\r
 \r
 \r
-static void\r
-ipoib_destroy_port(\r
-       IN                              ipoib_adapter_t* const          p_adapter );\r
-\r
-\r
 void\r
 ipoib_join_mcast(\r
        IN                              ipoib_adapter_t* const          p_adapter );\r
@@ -274,6 +274,9 @@ adapter_construct(
        cl_spinlock_construct( &p_adapter->recv_stat_lock );\r
        cl_qpool_construct( &p_adapter->item_pool );\r
        KeInitializeMutex( &p_adapter->mutex, 0 );\r
+\r
+       cl_thread_construct(&p_adapter->destroy_thread);\r
+       \r
        cl_vector_construct( &p_adapter->ip_vector );\r
 \r
        cl_perf_construct( &p_adapter->perf );\r
@@ -337,6 +340,7 @@ adapter_init(
                return IB_ERROR;\r
        }\r
 \r
+\r
        /* Validate the port GUID and generate the MAC address. */\r
        status =\r
                ipoib_mac_from_guid( p_adapter->guids.port_guid, &p_adapter->mac );\r
@@ -348,6 +352,8 @@ adapter_init(
                return status;\r
        }\r
 \r
+       \r
+       \r
        /* Open AL. */\r
        status = p_adapter->p_ifc->open_al( &p_adapter->h_al );\r
        if( status != IB_SUCCESS )\r
@@ -377,7 +383,7 @@ __ipoib_pnp_reg(
        CL_ASSERT( !p_adapter->registering );\r
 \r
        p_adapter->registering = TRUE;\r
-\r
+       \r
        /* Register for PNP events. */\r
        cl_memclr( &pnp_req, sizeof(pnp_req) );\r
        pnp_req.pnp_class = IB_PNP_PORT | flags;\r
@@ -812,14 +818,31 @@ __ipoib_pnp_dereg(
        IN                              void*                                           context )\r
 {\r
        ipoib_adapter_t*        p_adapter;\r
-       ipoib_port_t*           p_port;\r
-       ib_api_status_t         status;\r
-       ib_pnp_event_t          state;\r
 \r
        IPOIB_ENTER( IPOIB_DBG_INIT );\r
 \r
        p_adapter = PARENT_STRUCT( context, ipoib_adapter_t, obj );\r
 \r
+       cl_thread_init(&p_adapter->destroy_thread, __ipoib_adapter_reset, (void*)p_adapter, "destroy_thread");\r
+       \r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
+\r
+}\r
+\r
+static void\r
+__ipoib_adapter_reset(\r
+       IN                              void*   context)\r
+{\r
+\r
+       ipoib_adapter_t *p_adapter;\r
+       ipoib_port_t            *p_port;\r
+       ib_api_status_t         status;\r
+       ib_pnp_event_t          state;\r
+       \r
+       IPOIB_ENTER( IPOIB_DBG_INIT );\r
+\r
+       p_adapter = (ipoib_adapter_t*)context;\r
+       \r
        /* Synchronize with destruction */\r
        KeWaitForMutexObject(\r
                &p_adapter->mutex, Executive, KernelMode, FALSE, NULL );\r

diff --git a/trunk/ulp/ipoib/kernel/ipoib_adapter.h b/trunk/ulp/ipoib/kernel/ipoib_adapter.h
index 34cc776437c9c9f5555111452248096445fe9e41..58de4eff6fb2ce450e752289b049ca814fa1ebb5 100644 (file)
--- a/trunk/ulp/ipoib/kernel/ipoib_adapter.h
+++ b/trunk/ulp/ipoib/kernel/ipoib_adapter.h
@@ -177,6 +177,7 @@ typedef struct _ipoib_adapter
 \r
        KMUTEX                                  mutex;\r
 \r
+       cl_thread_t                             destroy_thread;\r
        cl_vector_t                             ip_vector;\r
 \r
        cl_perf_t                               perf;\r

diff --git a/trunk/ulp/ipoib/kernel/ipoib_debug.h b/trunk/ulp/ipoib/kernel/ipoib_debug.h
index 7c9852a61663330623d464c4d8c51e67159faaf2..6a5e3c7ad02fd5ebe68c4076c518dd7a2b90f779 100644 (file)
--- a/trunk/ulp/ipoib/kernel/ipoib_debug.h
+++ b/trunk/ulp/ipoib/kernel/ipoib_debug.h
@@ -260,6 +260,7 @@ enum ref_cnt_buckets
        ref_get_bcast,\r
        ref_bcast,              /* join and create, used as base only */\r
        ref_join_mcast,\r
+       ref_leave_mcast,\r
        ref_endpt_track,        /* used when endpt is in port's child list. */\r
 \r
        ref_array_size, /* Used to size the array of ref buckets. */\r

diff --git a/trunk/ulp/ipoib/kernel/ipoib_endpoint.c b/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
index 55fda0562379b7d204e24af425c1dbdf8e7c93ac..d727b02eeb2f49d59c7bf60af1523203cb61822e 100644 (file)
--- a/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
+++ b/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
@@ -207,6 +207,16 @@ __endpt_destroying(
                        p_port->p_adapter->h_al, p_endpt->h_query );\r
                p_endpt->h_query = NULL;\r
        }\r
+\r
+       /* Leave the multicast group if it exists. */\r
+       if( p_endpt->h_mcast )\r
+       {\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,\r
+                       ("Leaving MCast group\n") );\r
+               ipoib_port_ref(p_port, ref_leave_mcast);\r
+               p_port->p_adapter->p_ifc->leave_mcast( p_endpt->h_mcast, ipoib_leave_mcast_cb );\r
+       }\r
+       \r
        cl_obj_unlock( p_obj );\r
        \r
        IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
@@ -225,14 +235,6 @@ __endpt_cleanup(
        p_endpt = PARENT_STRUCT( p_obj, ipoib_endpt_t, obj );\r
        p_port = __endpt_parent( p_endpt );\r
 \r
-       /* Leave the multicast group if it exists. */\r
-       if( p_endpt->h_mcast )\r
-       {\r
-               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,\r
-                       ("Leaving MCast group\n") );\r
-               p_port->p_adapter->p_ifc->leave_mcast( p_endpt->h_mcast, NULL );\r
-       }\r
-\r
        /* Destroy the AV if it exists. */\r
        if( p_endpt->h_av )\r
                p_port->p_adapter->p_ifc->destroy_av( p_endpt->h_av );\r

diff --git a/trunk/ulp/ipoib/kernel/ipoib_port.c b/trunk/ulp/ipoib/kernel/ipoib_port.c
index 70e80f26b5ec5a6c839a7dd4ff556d0f65ec920d..29b299a77228cccc48abe08fd2fdac69ea468d0f 100644 (file)
--- a/trunk/ulp/ipoib/kernel/ipoib_port.c
+++ b/trunk/ulp/ipoib/kernel/ipoib_port.c
@@ -449,6 +449,10 @@ static void
 __mcast_cb(\r
        IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
 \r
+void\r
+__leave_error_mcast_cb(\r
+       IN                              void                            *context );\r
+\r
 \r
 static intn_t\r
 __gid_cmp(\r
@@ -573,7 +577,8 @@ __port_construct(
        __endpt_mgr_construct( p_port );\r
 \r
        KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
-\r
+       KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
+       \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -4047,8 +4052,10 @@ __endpt_mgr_reset_all(
        IN                              ipoib_port_t* const                     p_port )\r
 {\r
        cl_map_item_t   *p_item;\r
-       ipoib_endpt_t   *p_endpt;\r
-       cl_qlist_t              mc_list;\r
+       ipoib_endpt_t           *p_endpt;\r
+       cl_qlist_t                      mc_list;\r
+       uint32_t                        local_exist = 0;\r
+\r
 \r
        IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
 \r
@@ -4070,6 +4077,7 @@ __endpt_mgr_reset_all(
                \r
                cl_qlist_insert_head(\r
                        &mc_list, &p_port->p_local_endpt->mac_item.pool_item.list_item );\r
+               local_exist = 1;\r
 \r
                p_port->p_local_endpt = NULL;\r
        }\r
@@ -4108,8 +4116,22 @@ __endpt_mgr_reset_all(
                }\r
                \r
        }\r
+\r
        cl_obj_unlock( &p_port->obj );\r
 \r
+\r
+       if(cl_qlist_count( &mc_list ) - local_exist)\r
+       {\r
+               p_port->mcast_cnt =  (uint32_t)cl_qlist_count( &mc_list ) - local_exist;\r
+       }\r
+       else\r
+       {\r
+               p_port->mcast_cnt = 0;\r
+               KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE );\r
+       }       \r
+\r
+       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt - local_exist));\r
+\r
        /* Destroy all multicast endpoints now that we have released the lock. */\r
        while( cl_qlist_count( &mc_list ) )\r
        {\r
@@ -5109,9 +5131,14 @@ ipoib_port_down(
                return;\r
        }\r
 \r
+       KeResetEvent(&p_port->leave_mcast_event);\r
+\r
        /* Reset all endpoints so we don't flush our ARP cache. */\r
        __endpt_mgr_reset_all( p_port );\r
 \r
+       KeWaitForSingleObject(\r
+               &p_port->leave_mcast_event, Executive, KernelMode, FALSE, NULL );\r
+       \r
        cl_obj_lock( &p_port->p_adapter->obj );\r
        ipoib_dereg_addrs( p_port->p_adapter );\r
        cl_obj_unlock( &p_port->p_adapter->obj );\r
@@ -5136,8 +5163,11 @@ __bcast_cb(
        {\r
                cl_obj_unlock( &p_port->obj );\r
                if( p_mcast_rec->status == IB_SUCCESS )\r
-                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
 \r
+               {\r
+                       ipoib_port_ref(p_port, ref_leave_mcast);\r
+                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               }\r
                KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );\r
                ipoib_port_deref( p_port, ref_bcast_inv_state );\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
@@ -5197,7 +5227,8 @@ __bcast_cb(
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("__endpt_mgr_add_bcast returned %s\n",\r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
+               ipoib_port_ref(p_port, ref_leave_mcast);\r
+               status = p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
                CL_ASSERT( status == IB_SUCCESS );\r
                goto err;\r
        }\r
@@ -5453,8 +5484,11 @@ __mcast_cb(
        {\r
                cl_obj_unlock( &p_port->obj );\r
                if( p_mcast_rec->status == IB_SUCCESS )\r
-                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
 \r
+               {\r
+                       ipoib_port_ref(p_port, ref_leave_mcast);\r
+                       p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
+               }\r
                ipoib_port_deref( p_port, ref_mcast_inv_state );\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
                        ("Invalid state - Aborting.\n") );\r
@@ -5487,7 +5521,9 @@ __mcast_cb(
                cl_obj_unlock( &p_port->obj );\r
                IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,\r
                        ("Failed to find endpoint for update.\n") );\r
-               p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );\r
+\r
+               ipoib_port_ref(p_port, ref_leave_mcast);\r
+               p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, __leave_error_mcast_cb );\r
                ipoib_port_deref( p_port, ref_mcast_no_endpt );\r
                IPOIB_EXIT( IPOIB_DBG_MCAST );\r
                return;\r
@@ -5531,3 +5567,51 @@ __mcast_cb(
 \r
        IPOIB_EXIT( IPOIB_DBG_MCAST );\r
 }\r
+\r
+\r
+void\r
+ipoib_leave_mcast_cb(\r
+       IN                              void                            *context )\r
+{\r
+       ipoib_port_t            *p_port;\r
+\r
+       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+\r
+       p_port = (ipoib_port_t* __ptr64)context;\r
+\r
+       IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,("p_port->mcast_cnt = %d\n", p_port->mcast_cnt));\r
+       \r
+       ipoib_port_deref( p_port, ref_leave_mcast);\r
+       cl_atomic_dec( &p_port->mcast_cnt);\r
+       \r
+       if(0 == p_port->mcast_cnt)\r
+       {\r
+               KeSetEvent( &p_port->leave_mcast_event, EVENT_INCREMENT, FALSE );\r
+       }\r
+       \r
+       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST,\r
+                       ("Leav mcast callback deref ipoib_port \n") );\r
+       \r
+       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+}\r
+\r
+\r
+\r
+void\r
+__leave_error_mcast_cb(\r
+       IN                              void                            *context )\r
+{\r
+       ipoib_port_t            *p_port;\r
+\r
+       IPOIB_ENTER( IPOIB_DBG_MCAST );\r
+\r
+       p_port = (ipoib_port_t* __ptr64)context;\r
+\r
+       ipoib_port_deref( p_port, ref_leave_mcast);\r
+       IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST,\r
+                       ("Leav mcast callback deref ipoib_port \n") );\r
+       \r
+       IPOIB_EXIT( IPOIB_DBG_MCAST );\r
+}\r
+\r
+\r

diff --git a/trunk/ulp/ipoib/kernel/ipoib_port.h b/trunk/ulp/ipoib/kernel/ipoib_port.h
index 046d150062ef731323edeb1a2b6a411dc81323b6..443ec61a6c5ea6648117d58fefe43b6078e790a4 100644 (file)
--- a/trunk/ulp/ipoib/kernel/ipoib_port.h
+++ b/trunk/ulp/ipoib/kernel/ipoib_port.h
@@ -484,6 +484,10 @@ typedef struct _ipoib_port
        uint8_t                                 port_num;\r
 \r
        KEVENT                                  sa_event;\r
+\r
+       atomic32_t                              mcast_cnt;\r
+       KEVENT                                  leave_mcast_event;\r
+       \r
        ipoib_ib_mgr_t                  ib_mgr;\r
 \r
        ipoib_buf_mgr_t                 buf_mgr;\r
@@ -568,6 +572,12 @@ ipoib_port_join_mcast(
        IN              const   mac_addr_t                              mac,\r
        IN              const   uint8_t                                 state );\r
 \r
+\r
+void\r
+ipoib_leave_mcast_cb(\r
+       IN                              void                            *context );\r
+\r
+\r
 void\r
 ipoib_port_remove_endpt(\r
        IN                              ipoib_port_t* const                     p_port,\r