From: sleybo <sleybo@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Date: Thu, 13 Jul 2006 13:02:01 +0000 (+0000)
Subject: [IPOIB] bugfix: endpt are not added to the port child list if port is marked as being... 
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=bf642c0f3ddb222c56056df33a77053696f044d2;p=~shefty%2Frdma-win.git

[IPOIB] bugfix: endpt are not added to the port child list if port is marked as being destroy.
-add debug util to ref count
-port_resume is being called from port_destroy (and not from endpt_cleanup)

git-svn-id: svn://openib.tc.cornell.edu/gen1@421 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86
---

diff --git a/trunk/ulp/ipoib/kernel/ipoib_adapter.c b/trunk/ulp/ipoib/kernel/ipoib_adapter.c
index 9d99db63..d31380e3 100644
--- a/trunk/ulp/ipoib/kernel/ipoib_adapter.c
+++ b/trunk/ulp/ipoib/kernel/ipoib_adapter.c
@@ -717,7 +717,7 @@ ipoib_refresh_mcast(
 	if( p_adapter->state == IB_PNP_PORT_ACTIVE )
 	{
 		p_port = p_adapter->p_port;
-		cl_obj_ref( &p_port->obj );
+		ipoib_port_ref( p_port, ref_refresh_mcast );
 	}
 	cl_obj_unlock( &p_adapter->obj );
 
@@ -765,7 +765,7 @@ ipoib_refresh_mcast(
 	p_adapter->mcast_array_size = num_macs;
 
 	if( p_port )
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_refresh_mcast );
 
 	IPOIB_EXIT( IPOIB_DBG_MCAST );
 }
diff --git a/trunk/ulp/ipoib/kernel/ipoib_debug.h b/trunk/ulp/ipoib/kernel/ipoib_debug.h
index dc27bad6..20cc1b40 100644
--- a/trunk/ulp/ipoib/kernel/ipoib_debug.h
+++ b/trunk/ulp/ipoib/kernel/ipoib_debug.h
@@ -68,7 +68,8 @@ extern uint32_t		g_ipoib_dbg_flags;
 	WPP_DEFINE_BIT(IPOIB_DBG_ALLOC) \
 	WPP_DEFINE_BIT(IPOIB_DBG_OID) \
 	WPP_DEFINE_BIT(IPOIB_DBG_IOCTL) \
-	WPP_DEFINE_BIT(IPOIB_DBG_STAT))
+	WPP_DEFINE_BIT(IPOIB_DBG_STAT) \
+	WPP_DEFINE_BIT(IPOIB_DBG_OBJ))
 
 
 
@@ -111,6 +112,7 @@ extern uint32_t		g_ipoib_dbg_flags;
 #define IPOIB_DBG_OID	(1 << 10)
 #define IPOIB_DBG_IOCTL	(1 << 11)
 #define IPOIB_DBG_STAT	(1 << 12)
+#define IPOIB_DBG_OBJ	(1 << 13)
 
 #define IPOIB_DBG_ERROR	(CL_DBG_ERROR | IPOIB_DBG_ERR)
 #define IPOIB_DBG_ALL	CL_DBG_ALL
@@ -243,4 +245,49 @@ enum ipoib_perf_counters
 
 };
 
+
+enum ref_cnt_buckets
+{
+	ref_init = 0,
+	ref_refresh_mcast,	/* only used in refresh_mcast */
+	ref_send_packets,	/* only in send_packets */
+	ref_get_recv,
+	ref_repost,		/* only in __recv_mgr_repost */
+	ref_recv_cb,	/* only in __recv_cb */
+	ref_send_cb,	/* only in __send_cb */
+	ref_port_up,
+	ref_get_bcast,
+	ref_bcast,		/* join and create, used as base only */
+	ref_join_mcast,
+	ref_endpt_track,	/* used when endpt is in port's child list. */
+
+	ref_array_size,	/* Used to size the array of ref buckets. */
+	ref_mask = 100,	/* Used to differentiate derefs. */
+
+	ref_failed_recv_wc = 100 | ref_get_recv,
+	ref_recv_inv_len = 200 | ref_get_recv,
+	ref_recv_loopback = 300 | ref_get_recv,
+	ref_recv_filter = 400 | ref_get_recv,
+
+	ref_bcast_get_cb = 100 | ref_get_bcast,
+
+	ref_join_bcast = 100 | ref_bcast,
+	ref_create_bcast = 200 | ref_bcast,
+	ref_bcast_inv_state = 300 | ref_bcast,
+	ref_bcast_req_failed = 400 | ref_bcast,
+	ref_bcast_error = 500 | ref_bcast,
+	ref_bcast_join_failed = 600 | ref_bcast,
+	ref_bcast_create_failed = 700 | ref_bcast,
+
+	ref_mcast_inv_state = 100 | ref_join_mcast,
+	ref_mcast_req_failed = 200 | ref_join_mcast,
+	ref_mcast_no_endpt = 300 | ref_join_mcast,
+	ref_mcast_av_failed = 400 | ref_join_mcast,
+	ref_mcast_join_failed = 500 | ref_join_mcast,
+
+	ref_port_info_cb = 100 | ref_port_up
+
+};
+
+
 #endif	/* _IPOIB_DEBUG_H_ */
diff --git a/trunk/ulp/ipoib/kernel/ipoib_driver.c b/trunk/ulp/ipoib/kernel/ipoib_driver.c
index 08e9e85b..521c1265 100644
--- a/trunk/ulp/ipoib/kernel/ipoib_driver.c
+++ b/trunk/ulp/ipoib/kernel/ipoib_driver.c
@@ -1855,13 +1855,13 @@ ipoib_send_packets(
 	}
 
 	p_port = p_adapter->p_port;
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_send_packets );
 	cl_obj_unlock( &p_adapter->obj );
 
 	cl_perf_start( PortSend );
 	ipoib_port_send( p_port, packet_array, num_packets );
 	cl_perf_stop( &p_port->p_adapter->perf, PortSend );
-	cl_obj_deref( &p_port->obj );
+	ipoib_port_deref( p_port, ref_send_packets );
 
 	cl_perf_stop( &p_adapter->perf, SendPackets );
 
diff --git a/trunk/ulp/ipoib/kernel/ipoib_endpoint.c b/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
index 4ef32db4..5d885c96 100644
--- a/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
+++ b/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
@@ -224,9 +224,6 @@ __endpt_cleanup(
 	p_endpt = PARENT_STRUCT( p_obj, ipoib_endpt_t, obj );
 	p_port = __endpt_parent( p_endpt );
 
-	/* Resume sends so that any pending sends for this endpoint fail. */
-	ipoib_port_resume( p_port );
-
 	/* Leave the multicast group if it exists. */
 	if( p_endpt->h_mcast )
 	{
diff --git a/trunk/ulp/ipoib/kernel/ipoib_port.c b/trunk/ulp/ipoib/kernel/ipoib_port.c
index 3eca1335..e6c2fa34 100644
--- a/trunk/ulp/ipoib/kernel/ipoib_port.c
+++ b/trunk/ulp/ipoib/kernel/ipoib_port.c
@@ -389,13 +389,13 @@ __endpt_mgr_get_by_lid(
 	IN				ipoib_port_t* const			p_port,
 	IN		const	net16_t						lid );
 
-static inline void
+static inline ib_api_status_t
 __endpt_mgr_insert_locked(
 	IN				ipoib_port_t* const			p_port,
 	IN		const	mac_addr_t					mac,
 	IN				ipoib_endpt_t* const		p_endpt );
 
-static inline void
+static inline ib_api_status_t
 __endpt_mgr_insert(
 	IN				ipoib_port_t* const			p_port,
 	IN		const	mac_addr_t					mac,
@@ -458,6 +458,29 @@ __gid_cmp(
 }
 
 
+inline void ipoib_port_ref( ipoib_port_t * p_port, int type )
+{
+	cl_obj_ref( &p_port->obj );
+#if DBG
+	cl_atomic_inc( &p_port->ref[type % ref_mask] );
+	IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,
+		("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );
+#endif
+}
+
+
+inline void ipoib_port_deref(ipoib_port_t * p_port, int type)
+{
+	cl_obj_deref( &p_port->obj );
+
+#if DBG
+	cl_atomic_dec( &p_port->ref[type % ref_mask] );
+	IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,
+		("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );
+#endif
+}
+
+
 /******************************************************************************
 *
 * Implementation
@@ -623,6 +646,13 @@ __port_init(
 	/* We only ever destroy from the PnP callback thread. */
 	cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,
 		__port_destroying, __port_cleanup, __port_free );
+
+#if DBG
+	cl_atomic_inc( &p_port->ref[ref_init] );
+	IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,
+		("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );
+#endif
+
 	if( cl_status != CL_SUCCESS )
 	{
 		IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
@@ -630,7 +660,20 @@ __port_init(
 		return IB_ERROR;
 	}
 
-	cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );
+	cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );
+	if( cl_status != CL_SUCCESS )
+	{
+		IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
+			("cl_obj_insert_rel returned %s\n", cl_status_text[cl_status]) );
+		cl_obj_destroy( &p_port->obj );
+		return IB_ERROR;
+	}
+
+#if DBG
+	cl_atomic_inc( &p_port->ref[ref_init] );
+	IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,
+		("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );
+#endif
 
 	IPOIB_EXIT( IPOIB_DBG_INIT );
 	return IB_SUCCESS;
@@ -653,6 +696,8 @@ __port_destroying(
 
 	__endpt_mgr_remove_all( p_port );
 
+	ipoib_port_resume( p_port );
+
 	IPOIB_EXIT( IPOIB_DBG_INIT );
 }
 
@@ -1137,7 +1182,7 @@ __buf_mgr_get_recv(
 	/* Reference the port object for the send. */
 	if( p_desc )
 	{
-		cl_obj_ref( &p_port->obj );
+		ipoib_port_ref( p_port, ref_get_recv );
 		CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );
 #if IPOIB_INLINE_RECV
 		CL_ASSERT( p_desc->local_ds[0].vaddr ==
@@ -1181,8 +1226,7 @@ __buf_mgr_put_recv(
 	/*
 	 * Dereference the port object since the receive is no longer outstanding.
 	 */
-	cl_obj_deref( &p_port->obj );
-
+	ipoib_port_deref( p_port, ref_get_recv );
 	IPOIB_EXIT(  IPOIB_DBG_RECV );
 }
 
@@ -1327,7 +1371,7 @@ __recv_mgr_repost(
 			("Port in invalid state.  Not reposting.\n") );
 		return 0;
 	}
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_repost );
 	cl_obj_unlock( &p_port->obj );
 
 	while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )
@@ -1382,8 +1426,7 @@ __recv_mgr_repost(
 		}
 	}
 
-	cl_obj_deref( &p_port->obj );
-
+	ipoib_port_deref( p_port, ref_repost );
 	IPOIB_EXIT( IPOIB_DBG_RECV );
 	return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;
 }
@@ -1513,8 +1556,7 @@ __recv_cb(
 	cl_qlist_init( &done_list );
 	cl_qlist_init( &bad_list );
 
-	cl_obj_ref( &p_port->obj );
-
+	ipoib_port_ref( p_port, ref_recv_cb );
 	for( i = 0; i < MAX_RECV_WC; i++ )
 		wc[i].p_next = &wc[i + 1];
 	wc[MAX_RECV_WC - 1].p_next = NULL;
@@ -1633,7 +1675,7 @@ __recv_cb(
 	cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );
 	CL_ASSERT( status == IB_SUCCESS );
 
-	cl_obj_deref( &p_port->obj );
+	ipoib_port_deref( p_port, ref_recv_cb );
 
 	cl_perf_stop( &p_port->p_adapter->perf, RecvCb );
 
@@ -1715,13 +1757,19 @@ __recv_get_endpts(
 			if( !*pp_src )
 			{
 				IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
-					("ipoib_endpt_create returned %s\n",
-					p_port->p_adapter->p_ifc->get_err_str( status )) );
+					("ipoib_endpt_create failed\n") );
 				return;
 			}
 			cl_perf_start( EndptInsert );
 			cl_obj_lock( &p_port->obj );
-			__endpt_mgr_insert( p_port, mac, *pp_src );
+			status = __endpt_mgr_insert( p_port, mac, *pp_src );
+			if( status != IB_SUCCESS )
+			{
+				IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
+					("__endpt_mgr_insert returned %s\n",
+					p_port->p_adapter->p_ifc->get_err_str( status )) );
+				return;
+			}
 			cl_obj_unlock( &p_port->obj );
 			cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );
 		}
@@ -1816,7 +1864,7 @@ __recv_mgr_filter(
 			}
 			cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );
 			/* Dereference the port object on behalf of the failed receive. */
-			cl_obj_deref( &p_port->obj );
+			ipoib_port_deref( p_port, ref_failed_recv_wc );
 			continue;
 		}
 
@@ -1828,7 +1876,7 @@ __recv_mgr_filter(
 				("Received ETH packet < min size\n") );
 			ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );
 			cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );
-			cl_obj_deref( &p_port->obj );
+			ipoib_port_deref( p_port, ref_recv_inv_len );
 			continue;
 		}
 
@@ -1857,7 +1905,7 @@ __recv_mgr_filter(
 				 */
 				cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );
 				/* Dereference the port object on behalf of the failed recv. */
-				cl_obj_deref( &p_port->obj );
+				ipoib_port_deref( p_port, ref_recv_loopback );
 				continue;
 			}
 		}
@@ -1948,7 +1996,7 @@ __recv_mgr_filter(
 			ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );
 			cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );
 			/* Dereference the port object on behalf of the failed receive. */
-			cl_obj_deref( &p_port->obj );
+			ipoib_port_deref( p_port, ref_recv_filter );
 		}
 		else
 		{
@@ -2255,16 +2303,24 @@ __recv_arp(
 		 */
 		*pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,
 			0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );
+
 		if( !*pp_src )
 		{
 			IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
-				("ipoib_endpt_create returned %s\n",
-				p_port->p_adapter->p_ifc->get_err_str( status )) );
+				("ipoib_endpt_create failed\n") );
 			return status;
 		}
 
 		cl_obj_lock( &p_port->obj );
-		__endpt_mgr_insert( p_port, mac, *pp_src );
+		status = __endpt_mgr_insert( p_port, mac, *pp_src );
+		if( status != IB_SUCCESS )
+		{
+			IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
+				("__endpt_mgr_insert return %s \n",
+				p_port->p_adapter->p_ifc->get_err_str( status )) );
+			return status;
+		}
+
 		cl_obj_unlock( &p_port->obj );
 	}
 
@@ -3774,7 +3830,7 @@ __send_cb(
 
 	p_port = (ipoib_port_t*)cq_context;
 
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_send_cb );
 
 	for( i = 0; i < MAX_SEND_WC; i++ )
 		wc[i].p_next = &wc[i + 1];
@@ -3873,7 +3929,7 @@ __send_cb(
 	ipoib_port_resume( p_port );
 	cl_perf_stop( &p_port->p_adapter->perf, PortResume );
 	
-	cl_obj_deref( &p_port->obj );
+	ipoib_port_deref( p_port, ref_send_cb );
 
 	cl_perf_stop( &p_port->p_adapter->perf, SendCb );
 	cl_perf_update_ctr( &p_port->p_adapter->perf, SendCompBundle );
@@ -4240,12 +4296,14 @@ __endpt_mgr_get_by_lid(
 }
 
 
-inline void
+inline ib_api_status_t
 __endpt_mgr_insert_locked(
 	IN				ipoib_port_t* const			p_port,
 	IN		const	mac_addr_t					mac,
 	IN				ipoib_endpt_t* const		p_endpt )
 {
+	ib_api_status_t	status;
+
 	IPOIB_ENTER( IPOIB_DBG_ENDPT );
 
 	IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,
@@ -4261,21 +4319,22 @@ __endpt_mgr_insert_locked(
 	}
 	/* __endpt_mgr_insert expects *one* reference to be held when being called. */
 	cl_atomic_inc( &p_port->endpt_rdr );
-	__endpt_mgr_insert( p_port, mac, p_endpt );
+	status= __endpt_mgr_insert( p_port, mac, p_endpt );
 	cl_atomic_dec( &p_port->endpt_rdr );
 	cl_obj_unlock( &p_port->obj );
 
-	IPOIB_EXIT( IPOIB_DBG_ENDPT );
+	return status;
 }
 
 
-void
+inline ib_api_status_t
 __endpt_mgr_insert(
 	IN				ipoib_port_t* const			p_port,
 	IN		const	mac_addr_t					mac,
 	IN				ipoib_endpt_t* const		p_endpt )
 {
 	uint64_t		key;
+	cl_status_t		cl_status;
 	cl_map_item_t	*p_qitem;
 	cl_fmap_item_t	*p_fitem;
 
@@ -4286,9 +4345,21 @@ __endpt_mgr_insert(
 		;
 
 	/* Link the endpoint to the port. */
-	cl_obj_insert_rel_parent_locked(
+	cl_status = cl_obj_insert_rel_parent_locked(
 		&p_endpt->rel, &p_port->obj, &p_endpt->obj );
 
+	if( cl_status != CL_SUCCESS )
+	{
+		cl_obj_destroy( &p_endpt->obj );
+		return IB_INVALID_STATE;
+	}
+
+#if DBG
+	cl_atomic_inc( &p_port->ref[ref_endpt_track] );
+	IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,
+		("ref  type %d ref_cnt %d\n", ref_endpt_track, p_port->obj.ref_cnt) );
+#endif
+
 	p_endpt->mac = mac;
 	key = 0;
 	cl_memcpy( &key, &mac, sizeof(mac_addr_t) );
@@ -4306,6 +4377,7 @@ __endpt_mgr_insert(
 	}
 
 	IPOIB_EXIT( IPOIB_DBG_ENDPT );
+	return IB_SUCCESS;
 }
 
 
@@ -4348,10 +4420,10 @@ __endpt_mgr_add_bcast(
 
 	/* Add the broadcast endpoint to the endpoint map. */
 	cl_memset( &bcast_mac, 0xFF, sizeof(bcast_mac) );
-	__endpt_mgr_insert_locked( p_port, bcast_mac, p_endpt );
+	status = __endpt_mgr_insert_locked( p_port, bcast_mac, p_endpt );
 
 	IPOIB_EXIT( IPOIB_DBG_INIT );
-	return IB_SUCCESS;
+	return status;
 }
 
 
@@ -4397,6 +4469,12 @@ ipoib_port_remove_endpt(
 
 		cl_obj_unlock( &p_port->obj );
 		cl_obj_destroy( &p_endpt->obj );
+#if DBG
+		cl_atomic_dec( &p_port->ref[ref_endpt_track] );
+		IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,
+			("ref type %d ref_cnt %d\n", ref_endpt_track, p_port->obj.ref_cnt) );
+#endif
+
 	}
 	else
 	{
@@ -4506,7 +4584,7 @@ ipoib_port_up(
 	query.pfn_query_cb = __port_info_cb;
 
 	/* reference the object for the multicast query. */
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_port_up );
 
 	status = p_port->p_adapter->p_ifc->query(
 		p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );
@@ -4514,7 +4592,7 @@ ipoib_port_up(
 	{
 		KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );
 		ipoib_set_inactive( p_port->p_adapter );
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_port_up );
 		IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
 			("ib_query returned %s\n", 
 			p_port->p_adapter->p_ifc->get_err_str( status )) );
@@ -4567,13 +4645,20 @@ __endpt_mgr_add_local(
 
 	/* __endpt_mgr_insert expects *one* reference to be held. */
 	cl_atomic_inc( &p_port->endpt_rdr );
-	__endpt_mgr_insert( p_port, p_port->p_adapter->params.conf_mac, p_endpt );
+	status = __endpt_mgr_insert( p_port, p_port->p_adapter->params.conf_mac, p_endpt );
 	cl_atomic_dec( &p_port->endpt_rdr );
+	if( status != IB_SUCCESS )
+	{
+		IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
+			("__endpt_mgr_insert for local endpoint returned %s\n",
+			p_port->p_adapter->p_ifc->get_err_str( status )) );
+		return status;
+	}
 
 	p_port->p_local_endpt = p_endpt;
 
 	IPOIB_EXIT( IPOIB_DBG_INIT );
-	return IB_SUCCESS;
+	return status;
 }
 
 
@@ -4674,7 +4759,7 @@ done:
 		p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );
 
 	/* Release the reference taken when issuing the port info query. */
-	cl_obj_deref( &p_port->obj );
+	ipoib_port_deref( p_port, ref_port_info_cb );
 
 	IPOIB_EXIT( IPOIB_DBG_INIT );
 }
@@ -4711,13 +4796,13 @@ __port_get_bcast(
 	query.pfn_query_cb = __bcast_get_cb;
 
 	/* reference the object for the multicast query. */
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_get_bcast );
 
 	status = p_port->p_adapter->p_ifc->query(
 		p_port->p_adapter->h_al, &query, &p_port->ib_mgr.h_query );
 	if( status != IB_SUCCESS )
 	{
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_get_bcast );
 		IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
 			("ib_query returned %s\n", 
 			p_port->p_adapter->p_ifc->get_err_str( status )) );
@@ -4796,7 +4881,7 @@ done:
 		p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );
 
 	/* Release the reference taken when issuing the member record query. */
-	cl_obj_deref( &p_port->obj );
+	ipoib_port_deref( p_port, ref_bcast_get_cb );
 
 	IPOIB_EXIT( IPOIB_DBG_MCAST );
 }
@@ -4855,13 +4940,13 @@ __port_join_bcast(
 	}
 
 	/* reference the object for the multicast join request. */
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_join_bcast );
 
 	status = p_port->p_adapter->p_ifc->join_mcast(
 		p_port->ib_mgr.h_qp, &mcast_req );
 	if( status != IB_SUCCESS )
 	{
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_bcast_join_failed );
 		IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
 			("ib_join_mcast returned %s\n", 
 			p_port->p_adapter->p_ifc->get_err_str( status )) );
@@ -4915,12 +5000,12 @@ __port_create_bcast(
 	mcast_req.pkey_index = 0;
 
 	/* reference the object for the multicast join request. */
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_join_bcast );
 
 	status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );
 	if( status != IB_SUCCESS )
 	{
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_bcast_create_failed );
 		IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
 			("ib_join_mcast returned %s\n", 
 			p_port->p_adapter->p_ifc->get_err_str( status )) );
@@ -5011,7 +5096,7 @@ __bcast_cb(
 			p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );
 
 		KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_bcast_inv_state );
 		IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
 			("Invalid state - Aborting.\n") );
 		return;
@@ -5056,7 +5141,7 @@ __bcast_cb(
 			ipoib_set_inactive( p_port->p_adapter );
 			KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );
 		}
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_bcast_req_failed );
 		IPOIB_EXIT( IPOIB_DBG_INIT );
 		return;
 	}
@@ -5085,7 +5170,7 @@ err:
 		/* Flag the adapter as hung. */
 		p_port->p_adapter->hung = TRUE;
 		KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_bcast_error );
 		IPOIB_EXIT( IPOIB_DBG_INIT );
 		return;
 	}
@@ -5105,7 +5190,7 @@ err:
 	ipoib_set_active( p_port->p_adapter );
 
 	KeSetEvent( &p_port->sa_event, EVENT_INCREMENT, FALSE );
-	cl_obj_deref( &p_port->obj );
+	ipoib_port_deref( p_port, ref_join_bcast );
 	IPOIB_EXIT( IPOIB_DBG_INIT );
 }
 
@@ -5277,15 +5362,22 @@ ipoib_port_join_mcast(
 		return IB_INSUFFICIENT_MEMORY;
 	}
 
-	__endpt_mgr_insert_locked( p_port, mac, p_endpt );
+	status = __endpt_mgr_insert_locked( p_port, mac, p_endpt );
+	if( status != IB_SUCCESS )
+	{
+		IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
+			("__endpt_mgr_insert_locked returned %s\n", 
+			p_port->p_adapter->p_ifc->get_err_str( status )) );
+		return status;
+	}
 
 	/* reference the object for the multicast join request. */
-	cl_obj_ref( &p_port->obj );
+	ipoib_port_ref( p_port, ref_join_mcast );
 
 	status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );
 	if( status != IB_SUCCESS )
 	{
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_mcast_join_failed );
 		IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
 			("ib_join_mcast returned %s\n", 
 			p_port->p_adapter->p_ifc->get_err_str( status )) );
@@ -5317,7 +5409,7 @@ __mcast_cb(
 		if( p_mcast_rec->status == IB_SUCCESS )
 			p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );
 
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_mcast_inv_state );
 		IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
 			("Invalid state - Aborting.\n") );
 		return;
@@ -5331,7 +5423,7 @@ __mcast_cb(
 			p_port->p_adapter->p_ifc->get_err_str( p_mcast_rec->status )) );
 		/* Flag the adapter as hung. */
 		p_port->p_adapter->hung =TRUE;
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_mcast_req_failed );
 		IPOIB_EXIT( IPOIB_DBG_MCAST );
 		return;
 	}
@@ -5350,7 +5442,7 @@ __mcast_cb(
 		IPOIB_PRINT(TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,
 			("Failed to find endpoint for update.\n") );
 		p_port->p_adapter->p_ifc->leave_mcast( p_mcast_rec->h_mcast, NULL );
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_mcast_no_endpt );
 		IPOIB_EXIT( IPOIB_DBG_MCAST );
 		return;
 	}
@@ -5369,7 +5461,7 @@ __mcast_cb(
 			p_port->p_adapter->p_ifc->get_err_str( status )) );
 		/* Flag the adapter as hung. */
 		p_port->p_adapter->hung = TRUE;
-		cl_obj_deref( &p_port->obj );
+		ipoib_port_deref( p_port, ref_mcast_av_failed );
 		IPOIB_EXIT( IPOIB_DBG_MCAST );
 		return;
 	}
@@ -5389,7 +5481,7 @@ __mcast_cb(
 	/* Try to send all pending sends. */
 	ipoib_port_resume( p_port );
 
-	cl_obj_deref( &p_port->obj );
+	ipoib_port_deref( p_port, ref_join_mcast );
 
 	IPOIB_EXIT( IPOIB_DBG_MCAST );
 }
diff --git a/trunk/ulp/ipoib/kernel/ipoib_port.h b/trunk/ulp/ipoib/kernel/ipoib_port.h
index b693ccc7..31ee612f 100644
--- a/trunk/ulp/ipoib/kernel/ipoib_port.h
+++ b/trunk/ulp/ipoib/kernel/ipoib_port.h
@@ -494,10 +494,14 @@ typedef struct _ipoib_port
 
 	ipoib_endpt_t			*p_local_endpt;
 
+#if DBG
+	atomic32_t				ref[ref_array_size];
+#endif
+
 	atomic32_t				endpt_rdr;
 
 	atomic32_t				hdr_idx;
-	ipoib_hdr_t				hdr[1];
+	ipoib_hdr_t				hdr[1];	/* Must be last! */
 
 }	ipoib_port_t;
 /*
@@ -592,4 +596,13 @@ ipoib_mac_to_gid(
 	IN		const	mac_addr_t					mac,
 		OUT			ib_gid_t*					p_gid );
 
+inline void ipoib_port_ref(
+	IN				ipoib_port_t *				p_port, 
+	IN				int						type);
+
+inline void ipoib_port_deref(
+	IN				ipoib_port_t *				p_port,
+	IN				int						type);
+
+
 #endif	/* _IPOIB_PORT_H_ */