From: sleybo <sleybo@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Date: Sun, 12 Nov 2006 09:50:34 +0000 (+0000)
Subject: [MTHCA\MT23108\IBAL] change to support TRAP and TRAP_REPRESS
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=9b676b0f5922fdfbe659ea5a32f47cba7556098f;p=~shefty%2Frdma-win.git

[MTHCA\MT23108\IBAL] change to support TRAP and TRAP_REPRESS
1. set TRAP_FORWARD in case slid == 0
2. fix command MAD_IFC to set the correct values
3. add al_local_mad function and change ci interface to pass av_src_attr for traps generation.
4. ib_local_mad support NULL pointer at p_mad_out
5. in case of response unexpected the response does not pass to the recv dispatcher
7. add sm_lid and sm_sl to svc structure

git-svn-id: svn://openib.tc.cornell.edu/gen1@539 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86
---

diff --git a/trunk/core/al/al_mad.c b/trunk/core/al/al_mad.c
index 693de6d1..842da45e 100644
--- a/trunk/core/al/al_mad.c
+++ b/trunk/core/al/al_mad.c
@@ -331,9 +331,10 @@ build_mad_recv(
 	p_mad_element->remote_sl	= p_wc->recv.ud.remote_sl;
 	p_mad_element->pkey_index	= p_wc->recv.ud.pkey_index;
 	p_mad_element->path_bits	= p_wc->recv.ud.path_bits;
-	p_mad_element->recv_opt		= p_wc->recv.ud.recv_opt;
-	p_mad_element->grh_valid	= p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID;
+	p_mad_element->recv_opt	= p_wc->recv.ud.recv_opt;
 
+	p_mad_element->grh_valid	= p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID;
+	
 	if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_IMMEDIATE )
 		p_mad_element->immediate_data = p_wc->recv.ud.immediate_data;
 
@@ -1511,10 +1512,10 @@ __init_send_mad(
 		}
 	}
 
-	/* See if we need to create the address vector for the user. */
-	if( !p_mad_element->h_av &&
-		!( p_mad_element->send_opt & IB_SEND_OPT_LOCAL ) )
-	{
+	/* See if we need to create the address vector for the user. 
+		We also create AV for local send to pass the slid and grh in case of trap generation*/
+	if( !p_mad_element->h_av){
+
 		status = __create_send_av( h_mad_svc, h_send );
 		if( status != IB_SUCCESS )
 		{
@@ -3182,13 +3183,58 @@ ib_local_mad(
 		return IB_INVALID_PARAMETER;
 	}
 
-	status = verbs_local_mad( h_ca, port_num, p_mad_in, p_mad_out );
+	status = al_local_mad(h_ca, port_num, NULL,p_mad_in, p_mad_out);
 
 	AL_EXIT( AL_DBG_MAD_SVC );
 	return status;
 }
 
+ib_api_status_t
+al_local_mad(	
+	IN		const	ib_ca_handle_t				h_ca,
+	IN		const	uint8_t						port_num,
+	IN		const	ib_av_attr_t*					p_src_av_attr,
+	IN		const	void* const					p_mad_in,
+	IN				void*						p_mad_out )
+{
+	ib_api_status_t			status;
+	void*					p_mad_out_local = NULL;
+	AL_ENTER( AL_DBG_MAD_SVC );
 
+	if( AL_OBJ_INVALID_HANDLE( h_ca, AL_OBJ_TYPE_H_CA ) )
+	{
+		AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INVALID_CA_HANDLE\n") );
+		return IB_INVALID_CA_HANDLE;
+	}
+	if( !p_mad_in )
+	{
+		AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INVALID_PARAMETER\n") );
+		return IB_INVALID_PARAMETER;
+	}
+	if( !p_mad_out )
+	{
+		p_mad_out_local = cl_zalloc(256);
+		if(!p_mad_out_local)
+		{
+			AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INSUFFICIENT_MEMORY\n") );
+			return IB_INSUFFICIENT_MEMORY;
+		}
+	}else
+	{
+		p_mad_out_local = p_mad_out;
+	}
+		
+	status = verbs_local_mad( h_ca, port_num, p_src_av_attr, p_mad_in, p_mad_out_local );
+	
+	if( !p_mad_out )
+	{
+		cl_free(p_mad_out_local);
+	}
+	
+	AL_EXIT( AL_DBG_MAD_SVC );
+	return status;
+	
+}
 
 ib_net32_t
 al_get_user_tid(
diff --git a/trunk/core/al/al_mad.h b/trunk/core/al/al_mad.h
index 380618cb..84bdeee2 100644
--- a/trunk/core/al/al_mad.h
+++ b/trunk/core/al/al_mad.h
@@ -175,6 +175,13 @@ reg_mad_svc(
 		OUT			ib_mad_svc_handle_t* const	ph_mad_svc );
 
 
+ib_api_status_t
+al_local_mad(	
+	IN		const	ib_ca_handle_t				h_ca,
+	IN		const	uint8_t						port_num,
+	IN		const	ib_av_attr_t*					p_av_attr,
+	IN		const	void* const					p_mad_in,
+	IN				void*						p_mad_out );
 
 /*
  * TID management
diff --git a/trunk/core/al/al_verbs.h b/trunk/core/al/al_verbs.h
index 33b3705f..f0b67fb6 100644
--- a/trunk/core/al/al_verbs.h
+++ b/trunk/core/al/al_verbs.h
@@ -304,9 +304,9 @@ verbs_create_qp(
 	h_qp->obj.p_ci_ca->verbs.post_recv( h_qp->h_ci_qp,\
 		p_recv_wr, pp_recv_failure )
 
-#define verbs_local_mad(h_ca, port_num, p_mad_in, p_mad_out) \
+#define verbs_local_mad(h_ca, port_num, p_src_av_attr, p_mad_in, p_mad_out) \
 	h_ca->obj.p_ci_ca->verbs.local_mad( h_ca->obj.p_ci_ca->h_ci_ca,\
-		port_num, p_mad_in, p_mad_out)
+		port_num, p_src_av_attr, p_mad_in, p_mad_out)
 
 #define check_local_mad(h_qp) \
 	(h_qp->obj.p_ci_ca->verbs.local_mad)
@@ -541,10 +541,12 @@ static inline ib_api_status_t
 verbs_local_mad(
 	IN		const	ib_ca_handle_t				h_ca,
 	IN		const	uint8_t						port_num,
+	IN		const	ib_av_attr_t*					p_src_av_attr,
 	IN		const	void* const					p_mad_in,
 	IN				void*						p_mad_out )
 {
 	return ual_local_mad( h_ca, port_num, p_mad_in, p_mad_out );
+	UNUSED_PARAM( p_src_av_attr );
 }
 
 #define check_local_mad(h_qp) \
diff --git a/trunk/core/al/kernel/al_smi.c b/trunk/core/al/kernel/al_smi.c
index d15944c6..ca4adb82 100644
--- a/trunk/core/al/kernel/al_smi.c
+++ b/trunk/core/al/kernel/al_smi.c
@@ -1514,7 +1514,25 @@ loopback_mad(
 		/* Simulate a send/receive between local managers. */
 		cl_memcpy( p_mad_resp->p_mad_buf, p_mad, MAD_BLOCK_SIZE );
 
-		status = complete_local_mad( p_spl_qp_svc, p_mad_wr, p_mad_resp );
+		/* Construct the receive MAD element. */
+		p_mad_resp->status		= IB_WCS_SUCCESS;
+		p_mad_resp->remote_qp	= p_mad_wr->send_wr.dgrm.ud.remote_qp;
+		p_mad_resp->remote_lid	= p_spl_qp_svc->base_lid;
+		if( p_mad_wr->send_wr.send_opt & IB_RECV_OPT_IMMEDIATE )
+		{
+			p_mad_resp->immediate_data = p_mad_wr->send_wr.immediate_data;
+			p_mad_resp->recv_opt |= IB_RECV_OPT_IMMEDIATE;
+		}
+
+		/*
+		 * Hand the receive MAD element to the dispatcher before completing
+		 * the send.  This guarantees that the send request cannot time out.
+		 */
+		status = mad_disp_recv_done( p_spl_qp_svc->h_mad_disp, p_mad_resp );
+
+		/* Forward the send work completion to the dispatcher. */
+		__complete_send_mad( p_spl_qp_svc->h_mad_disp, p_mad_wr, IB_WCS_SUCCESS );
+
 	}
 
 	AL_EXIT( AL_DBG_SMI );
@@ -1697,7 +1715,7 @@ fwd_local_mad(
 	ib_mad_t*				p_mad;
 	ib_smp_t*				p_smp;
 	al_mad_send_t*			p_mad_send;
-	ib_mad_element_t*		p_mad_response;
+	ib_mad_element_t*		p_mad_response = NULL;
 	ib_mad_t*				p_mad_response_buf;
 	ib_api_status_t			status = IB_SUCCESS;
 	boolean_t				smp_is_set;
@@ -1715,25 +1733,22 @@ fwd_local_mad(
 
 	/* Get a MAD element from the pool for the response. */
 	p_mad_send = PARENT_STRUCT( p_mad_wr, al_mad_send_t, mad_wr );
-//*** Commented code to work-around ib_local_mad() requiring a response MAD
-//*** as input.  Remove comments once the ib_local_mad() implementation allows
-//*** for a NULL response MAD, when one is not expected.
-//*** Note that an attempt to route an invalid response MAD in this case
-//*** will fail harmlessly.
-//***	if( p_mad_send->p_send_mad->resp_expected )
-//***	{
+	if( p_mad_send->p_send_mad->resp_expected )
+	{
 		status = get_resp_mad( p_spl_qp_svc, p_mad_wr, &p_mad_response );
 		if( status != IB_SUCCESS )
 		{
+			__complete_send_mad( p_spl_qp_svc->h_mad_disp, p_mad_wr,
+				IB_WCS_LOCAL_OP_ERR );
 			AL_EXIT( AL_DBG_SMI );
 			return status;
 		}
 		p_mad_response_buf = p_mad_response->p_mad_buf;
-//***	}
-//***	else
-//***	{
-//***		p_mad_response_buf = NULL;
-//***	}
+	}
+	else
+	{
+			p_mad_response_buf = NULL;
+	}
 
 	/* Adjust directed route SMPs as required by IBA. */
 	if( p_mad->mgmt_class == IB_MCLASS_SUBN_DIR )
@@ -1751,8 +1766,8 @@ fwd_local_mad(
 	}
 
 	/* Forward the locally addressed MAD to the CA interface. */
-	status = ib_local_mad( p_spl_qp_svc->h_qp->obj.p_ci_ca->h_ca,
-		p_spl_qp_svc->port_num, p_mad, p_mad_response_buf );
+	status = al_local_mad( p_spl_qp_svc->h_qp->obj.p_ci_ca->h_ca,
+		p_spl_qp_svc->port_num, &p_mad_wr->send_wr.dgrm.ud.h_av->av_attr, p_mad, p_mad_response_buf );
 
 	/* Reset directed route SMPs as required by IBA. */
 	if( p_mad->mgmt_class == IB_MCLASS_SUBN_DIR )
@@ -1789,7 +1804,7 @@ fwd_local_mad(
 	}
 
 	/* Check the completion status of this simulated send. */
-	if( p_mad_response_buf )
+	if( p_mad_send->p_send_mad->resp_expected )
 	{
 		/*
 		 * The SMI is uses PnP polling to refresh the base_lid and lmc.
@@ -1817,7 +1832,7 @@ fwd_local_mad(
 					( p_mad_response_buf->status == IB_SA_MAD_STATUS_SUCCESS ) )
 				{
 					p_port_info =
-						(ib_port_info_t*)( p_mad_response_buf + 1 );
+						(ib_port_info_t*)ib_smp_get_payload_ptr((ib_smp_t*)p_mad_response_buf);
 				}
 				break;
 
@@ -1826,9 +1841,12 @@ fwd_local_mad(
 			}
 
 			if( p_port_info )
-			{
+			{ 
 				p_spl_qp_svc->base_lid = p_port_info->base_lid;
 				p_spl_qp_svc->lmc = ib_port_info_get_lmc( p_port_info );
+				p_spl_qp_svc->sm_lid = p_port_info->master_sm_base_lid;
+				p_spl_qp_svc->sm_sl = ib_port_info_get_sm_sl( p_port_info );
+
 				if (p_port_info->subnet_timeout & 0x80)
 				{
 					AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_PNP,
@@ -1840,10 +1858,29 @@ fwd_local_mad(
 				}
 			}
 		}
-	}
+		
 
-	status = complete_local_mad( p_spl_qp_svc, p_mad_wr, p_mad_response );
+		/* Construct the receive MAD element. */
+		p_mad_response->status		= IB_WCS_SUCCESS;
+		p_mad_response->remote_qp	= p_mad_wr->send_wr.dgrm.ud.remote_qp;
+		p_mad_response->remote_lid	= p_spl_qp_svc->base_lid;
+		if( p_mad_wr->send_wr.send_opt & IB_RECV_OPT_IMMEDIATE )
+		{
+			p_mad_response->immediate_data = p_mad_wr->send_wr.immediate_data;
+			p_mad_response->recv_opt |= IB_RECV_OPT_IMMEDIATE;
+		}
 
+		/*
+		 * Hand the receive MAD element to the dispatcher before completing
+		 * the send.  This guarantees that the send request cannot time out.
+		 */
+		status = mad_disp_recv_done( p_spl_qp_svc->h_mad_disp, p_mad_response );
+	}
+	
+	__complete_send_mad( p_spl_qp_svc->h_mad_disp, p_mad_wr,IB_WCS_SUCCESS);
+
+	
+	
 	/* If the SMP was a Get, no need to trigger a PnP poll. */
 	if( status == IB_SUCCESS && !smp_is_set )
 		status = IB_NOT_DONE;
@@ -2518,17 +2555,11 @@ forward_sm_trap(
 	/* Create an address vector for the SM. */
 	cl_memclr( &av_attr, sizeof( ib_av_attr_t ) );
 	av_attr.port_num = p_spl_qp_svc->port_num;
-	av_attr.sl = p_mad_element->remote_sl;
-	av_attr.dlid = p_mad_element->remote_lid;
-	if( p_mad_element->grh_valid )
-	{
-		cl_memcpy( &av_attr.grh, p_mad_element->p_grh, sizeof( ib_grh_t ) );
-		av_attr.grh.src_gid	 = p_mad_element->p_grh->dest_gid;
-		av_attr.grh.dest_gid = p_mad_element->p_grh->src_gid;
-		av_attr.grh_valid = TRUE;
-	}
+	av_attr.sl = p_spl_qp_svc->sm_sl;
+	av_attr.dlid = p_spl_qp_svc->sm_lid;
+	av_attr.grh_valid = FALSE;
 
-	status = ib_create_av( p_spl_qp_svc->h_qp->obj.p_ci_ca->h_pd_alias,
+ 	status = ib_create_av( p_spl_qp_svc->h_qp->obj.p_ci_ca->h_pd_alias,
 		&av_attr, &p_mad_element->h_av );
 
 	if( status != IB_SUCCESS )
diff --git a/trunk/core/al/kernel/al_smi.h b/trunk/core/al/kernel/al_smi.h
index 9af6a080..e9f452d4 100644
--- a/trunk/core/al/kernel/al_smi.h
+++ b/trunk/core/al/kernel/al_smi.h
@@ -79,6 +79,9 @@ typedef struct _spl_qp_svc
 	ib_net16_t					base_lid;
 	uint8_t						lmc;
 
+	ib_net16_t					sm_lid;
+	uint8_t						sm_sl;
+
 	al_mad_disp_handle_t		h_mad_disp;
 	ib_cq_handle_t				h_send_cq;
 	ib_cq_handle_t				h_recv_cq;
diff --git a/trunk/hw/mt23108/kernel/hca_data.h b/trunk/hw/mt23108/kernel/hca_data.h
index 0f4e739a..ebf91d52 100644
--- a/trunk/hw/mt23108/kernel/hca_data.h
+++ b/trunk/hw/mt23108/kernel/hca_data.h
@@ -577,6 +577,7 @@ ib_api_status_t
 mlnx_local_mad (
 	IN		const	ib_ca_handle_t				h_ca,
 	IN		const	uint8_t						port_num,
+	IN		const	ib_av_attr_t					*p_av_src_attr,	
 	IN		const	ib_mad_t					*p_mad_in,
 		OUT			ib_mad_t					*p_mad_out );
 
diff --git a/trunk/hw/mt23108/kernel/hca_smp.c b/trunk/hw/mt23108/kernel/hca_smp.c
index a1ea245d..e57b5b06 100644
--- a/trunk/hw/mt23108/kernel/hca_smp.c
+++ b/trunk/hw/mt23108/kernel/hca_smp.c
@@ -491,8 +491,9 @@ ib_api_status_t
 mlnx_local_mad (
 	IN		const	ib_ca_handle_t				h_ca,
 	IN		const	uint8_t						port_num,
+	IN		const 	ib_av_attr_t					*p_av_src_attr,
 	IN		const	ib_mad_t					*p_mad_in,
-		OUT			ib_mad_t					*p_mad_out )
+	OUT			ib_mad_t					*p_mad_out )
 {
 	ib_api_status_t		status;
 
@@ -502,6 +503,7 @@ mlnx_local_mad (
 	HH_hca_dev_t		*hca_ul_info;
 
 	CL_ENTER(MLNX_DBG_TRACE, g_mlnx_dbg_lvl);
+	UNUSED_PARAM(*p_av_src_attr);
 
 	if (port_num > 2) {
 		status = IB_INVALID_PARAMETER;
diff --git a/trunk/hw/mthca/kernel/hca_data.h b/trunk/hw/mthca/kernel/hca_data.h
index 66681ce2..b3ce1da7 100644
--- a/trunk/hw/mthca/kernel/hca_data.h
+++ b/trunk/hw/mthca/kernel/hca_data.h
@@ -315,6 +315,7 @@ ib_api_status_t
 mlnx_local_mad (
 	IN		const	ib_ca_handle_t				h_ca,
 	IN		const	uint8_t						port_num,
+	IN		const	ib_av_attr_t				*p_src_av_attr,
 	IN		const	ib_mad_t					*p_mad_in,
 		OUT			ib_mad_t					*p_mad_out );
 
diff --git a/trunk/hw/mthca/kernel/hca_smp.c b/trunk/hw/mthca/kernel/hca_smp.c
index 22740062..6c52d69f 100644
--- a/trunk/hw/mthca/kernel/hca_smp.c
+++ b/trunk/hw/mthca/kernel/hca_smp.c
@@ -519,6 +519,7 @@ ib_api_status_t
 mlnx_local_mad (
 	IN		const	ib_ca_handle_t				h_ca,
 	IN		const	uint8_t						port_num,
+	IN		const	ib_av_attr_t*					p_av_attr,
 	IN		const	ib_mad_t					*p_mad_in,
 	OUT		ib_mad_t					*p_mad_out )
 {
@@ -528,9 +529,9 @@ mlnx_local_mad (
 	struct ib_device *ib_dev = IBDEV_FROM_HOB( hob_p );
 	//TODO: do we need use flags (IB_MAD_IGNORE_MKEY, IB_MAD_IGNORE_BKEY) ?
 	int mad_flags = 0;  
-	struct _ib_wc *wc_p;
+	struct _ib_wc *wc_p = NULL;
 	//TODO: do we need use grh ?
-	struct ib_grh *grh_p = NULL;
+	struct _ib_grh *grh_p = NULL;
 
 	HCA_ENTER(HCA_DBG_MAD);
 
@@ -540,18 +541,38 @@ mlnx_local_mad (
 		goto err_port_num;
 	}
 
-	wc_p = NULL;
-	
-	// debug print
-	{
-		HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_MAD, 
-			("MAD: Class %02x, Method %02x, Attr %02x, HopPtr %d, HopCnt %d, \n",
-			(uint32_t)((ib_smp_t *)p_mad_in)->mgmt_class, 
-			(uint32_t)((ib_smp_t *)p_mad_in)->method, 
-			(uint32_t)((ib_smp_t *)p_mad_in)->attr_id, 
-			(uint32_t)((ib_smp_t *)p_mad_in)->hop_ptr,
-			(uint32_t)((ib_smp_t *)p_mad_in)->hop_count));
+	if (p_av_attr){
+		wc_p = cl_zalloc(sizeof(struct _ib_wc));
+		if(!wc_p){
+			status =  IB_INSUFFICIENT_MEMORY ;
+			goto err_wc_alloc;
+		}
+		//Copy part of the attributes need to fill the mad extended fields in mellanox devices
+		wc_p->recv.ud.remote_lid = p_av_attr->dlid;
+		wc_p->recv.ud.remote_sl  = p_av_attr->sl;
+		wc_p->recv.ud.path_bits  = p_av_attr->path_bits;
+		wc_p->recv.ud.recv_opt = p_av_attr->grh_valid?IB_RECV_OPT_GRH_VALID:0;
+
+		if(wc_p->recv.ud.recv_opt &IB_RECV_OPT_GRH_VALID){
+			grh_p = cl_zalloc(sizeof(struct _ib_grh));
+			if(!grh_p){
+				status =  IB_INSUFFICIENT_MEMORY ;
+				goto err_grh_alloc;
+			}
+			cl_memcpy(grh_p, &p_av_attr->grh, sizeof(ib_grh_t));
+		}
+			
+
 	}
+
+	HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_MAD, 
+		("MAD: Class %02x, Method %02x, Attr %02x, HopPtr %d, HopCnt %d, \n",
+		(uint32_t)((ib_smp_t *)p_mad_in)->mgmt_class, 
+		(uint32_t)((ib_smp_t *)p_mad_in)->method, 
+		(uint32_t)((ib_smp_t *)p_mad_in)->attr_id, 
+		(uint32_t)((ib_smp_t *)p_mad_in)->hop_ptr,
+		(uint32_t)((ib_smp_t *)p_mad_in)->hop_count));
+
 	
 	// process mad
 	if( !mlnx_cachable_mad( h_ca, port_num, p_mad_in, p_mad_out ) )
@@ -598,7 +619,14 @@ mlnx_local_mad (
 	if ( p_mad_in->mgmt_class == IB_MCLASS_SUBN_DIR )
 		p_mad_out->status |= IB_SMP_DIRECTION;
 
+
 err_process_mad:
+	if(grh_p)
+		cl_free(grh_p);
+err_grh_alloc:
+	if(wc_p)
+		cl_free(wc_p);
+err_wc_alloc:
 err_port_num:	
 	HCA_PRINT_EXIT(TRACE_LEVEL_ERROR  , HCA_DBG_MAD,
 		("completes with ERROR status %s\n", IB_GET_ERR_STR(status)));
diff --git a/trunk/hw/mthca/kernel/ib_verbs.h b/trunk/hw/mthca/kernel/ib_verbs.h
index 2f83bf19..f8fe8925 100644
--- a/trunk/hw/mthca/kernel/ib_verbs.h
+++ b/trunk/hw/mthca/kernel/ib_verbs.h
@@ -813,7 +813,7 @@ struct ib_device {
 						  int process_mad_flags,
 						  u8 port_num,
 						  struct _ib_wc *in_wc,
-						  struct ib_grh *in_grh,
+						  struct _ib_grh *in_grh,
 						  struct ib_mad *in_mad,
 						  struct ib_mad *out_mad);
 
diff --git a/trunk/hw/mthca/kernel/mthca_cmd.c b/trunk/hw/mthca/kernel/mthca_cmd.c
index 62c156a5..ae2fad97 100644
--- a/trunk/hw/mthca/kernel/mthca_cmd.c
+++ b/trunk/hw/mthca/kernel/mthca_cmd.c
@@ -1705,7 +1705,7 @@ int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn,
 }
 
 int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
-		  int port, struct _ib_wc *in_wc, struct ib_grh *in_grh,
+		  int port, struct _ib_wc *in_wc, struct _ib_grh *in_grh,
 		  void *in_mad, void *response_mad, u8 *status)
 {
 	struct mthca_mailbox *inmailbox, *outmailbox;
@@ -1714,16 +1714,14 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
 	u32 in_modifier = port;
 	u8 op_modifier = 0;
 
-	ASSERT( !in_wc );
-	UNREFERENCED_PARAMETER( in_grh );
 
 #define MAD_IFC_BOX_SIZE      0x400
 #define MAD_IFC_MY_QPN_OFFSET 0x100
-#define MAD_IFC_RQPN_OFFSET   0x104
-#define MAD_IFC_SL_OFFSET     0x108
-#define MAD_IFC_G_PATH_OFFSET 0x109
-#define MAD_IFC_RLID_OFFSET   0x10a
-#define MAD_IFC_PKEY_OFFSET   0x10e
+#define MAD_IFC_RQPN_OFFSET   0x108
+#define MAD_IFC_SL_OFFSET     0x10c
+#define MAD_IFC_G_PATH_OFFSET 0x10d
+#define MAD_IFC_RLID_OFFSET   0x10e
+#define MAD_IFC_PKEY_OFFSET   0x112
 #define MAD_IFC_GRH_OFFSET    0x140
 
 	inmailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
@@ -1748,6 +1746,33 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
 	if (ignore_bkey || !in_wc)
 		op_modifier |= 0x2;
 
+	if (in_wc) {
+		u8 val;
+
+		memset(inbox + 256, 0, 256);
+
+
+		MTHCA_PUT(inbox, 0, MAD_IFC_MY_QPN_OFFSET);
+		MTHCA_PUT(inbox, cl_ntoh32(in_wc->recv.ud.remote_qp), MAD_IFC_RQPN_OFFSET);
+		val = in_wc->recv.ud.remote_sl << 4;
+		MTHCA_PUT(inbox, val, MAD_IFC_SL_OFFSET);
+
+		val = in_wc->recv.ud.path_bits |
+			(in_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID ? 0x80 : 0);
+		MTHCA_PUT(inbox, val, MAD_IFC_G_PATH_OFFSET)
+
+		MTHCA_PUT(inbox, cl_ntoh16(in_wc->recv.ud.remote_lid), MAD_IFC_RLID_OFFSET);
+		MTHCA_PUT(inbox, in_wc->recv.ud.pkey_index, MAD_IFC_PKEY_OFFSET);
+
+		if (in_grh)
+			memcpy(inbox + MAD_IFC_GRH_OFFSET, in_grh, 40);
+
+		op_modifier |= 0x4;
+		
+		in_modifier |= cl_ntoh16(in_wc->recv.ud.remote_lid) << 16;
+
+	}
+
 	err = mthca_cmd_box(dev, inmailbox->dma, outmailbox->dma,
 			    in_modifier, op_modifier,
 			    CMD_MAD_IFC, CMD_TIME_CLASS_C, status);
diff --git a/trunk/hw/mthca/kernel/mthca_cmd.h b/trunk/hw/mthca/kernel/mthca_cmd.h
index cfeec0c3..36584345 100644
--- a/trunk/hw/mthca/kernel/mthca_cmd.h
+++ b/trunk/hw/mthca/kernel/mthca_cmd.h
@@ -311,7 +311,7 @@ int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee,
 int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn,
 			  u8 *status);
 int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
-		  int port, struct _ib_wc *in_wc, struct ib_grh *in_grh,
+		  int port, struct _ib_wc *in_wc, struct _ib_grh *in_grh,
 		  void *in_mad, void *response_mad, u8 *status);
 int mthca_READ_MGM(struct mthca_dev *dev, int index,
 		   struct mthca_mailbox *mailbox, u8 *status);
diff --git a/trunk/hw/mthca/kernel/mthca_cq.c b/trunk/hw/mthca/kernel/mthca_cq.c
index 23192507..719e21ae 100644
--- a/trunk/hw/mthca/kernel/mthca_cq.c
+++ b/trunk/hw/mthca/kernel/mthca_cq.c
@@ -579,7 +579,11 @@ static inline int mthca_poll_one(struct mthca_dev *dev,
 		entry->recv.ud.recv_opt   |= cl_ntoh16(cqe->sl_g_mlpath) & 0x80 ?
 					IB_RECV_OPT_GRH_VALID : 0;
 	}
+	if (!is_send && cqe->rlid == 0){
+		HCA_PRINT(TRACE_LEVEL_INFORMATION,HCA_DBG_CQ,("found rlid == 0 \n "));
+		entry->recv.ud.recv_opt   |= IB_RECV_OPT_FORWARD;
 
+	}
 	if (is_error) {
 		handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send,
 			(struct mthca_err_cqe *) cqe, entry, &free_cqe);
diff --git a/trunk/hw/mthca/kernel/mthca_dev.h b/trunk/hw/mthca/kernel/mthca_dev.h
index 941931a6..4698c3b5 100644
--- a/trunk/hw/mthca/kernel/mthca_dev.h
+++ b/trunk/hw/mthca/kernel/mthca_dev.h
@@ -545,7 +545,7 @@ int mthca_process_mad(struct ib_device *ibdev,
 		      int mad_flags,
 		      u8 port_num,
 		      struct _ib_wc *in_wc,
-		      struct ib_grh *in_grh,
+		      struct _ib_grh *in_grh,
 		      struct ib_mad *in_mad,
 		      struct ib_mad *out_mad);
 
diff --git a/trunk/hw/mthca/kernel/mthca_mad.c b/trunk/hw/mthca/kernel/mthca_mad.c
index 87e6e3dc..07dee658 100644
--- a/trunk/hw/mthca/kernel/mthca_mad.c
+++ b/trunk/hw/mthca/kernel/mthca_mad.c
@@ -183,7 +183,7 @@ int mthca_process_mad(struct ib_device *ibdev,
 		      int mad_flags,
 		      u8 port_num,
 		      struct _ib_wc *in_wc,
-		      struct ib_grh *in_grh,
+		      struct _ib_grh *in_grh,
 		      struct ib_mad *in_mad,
 		      struct ib_mad *out_mad)
 {
diff --git a/trunk/inc/iba/ib_ci.h b/trunk/inc/iba/ib_ci.h
index 74b2b5d3..8941386d 100644
--- a/trunk/inc/iba/ib_ci.h
+++ b/trunk/inc/iba/ib_ci.h
@@ -73,7 +73,7 @@ extern "C"
  * definition.
  */
 #define VERBS_MAJOR_VER			(0x0001)
-#define VERBS_MINOR_VER			(0x0002)
+#define VERBS_MINOR_VER			(0x0003)
 
 #define VERBS_VERSION			(((VERBS_MAJOR_VER) << 16) | (VERBS_MINOR_VER))
 #define MK_VERBS_VERSION(maj,min)	((((maj) & 0xFFFF) << 16) | \
@@ -2323,8 +2323,9 @@ typedef ib_api_status_t
 (*ci_local_mad) (
 	IN		const	ib_ca_handle_t		h_ca,
 	IN		const	uint8_t				port_num,
+	IN		const	ib_av_attr_t			*p_src_av_attr,
 	IN		const	ib_mad_t			*p_mad_in,
-		OUT			ib_mad_t			*p_mad_out );
+	OUT			ib_mad_t			*p_mad_out );
 /*
 * DESCRIPTION
 *	This routine is OPTIONAL for the channel interface. This is required
diff --git a/trunk/inc/iba/ib_types.h b/trunk/inc/iba/ib_types.h
index 55ecc808..1a12dafa 100644
--- a/trunk/inc/iba/ib_types.h
+++ b/trunk/inc/iba/ib_types.h
@@ -4377,6 +4377,68 @@ ib_port_info_set_neighbor_mtu(
 * SEE ALSO
 *********/
 
+
+/****f* IBA Base: Types/ib_port_info_get_sm_sl
+* NAME
+*	ib_port_info_get_sm_sl
+*
+* DESCRIPTION
+*	Returns the encoded value for the SM sl at this port.
+*
+* SYNOPSIS
+*/
+AL_INLINE uint8_t AL_API
+ib_port_info_get_sm_sl(
+	IN const ib_port_info_t* const p_pi )
+{
+	return( (uint8_t)(p_pi->mtu_smsl & 0x0F) );
+}
+/*
+* PARAMETERS
+*	p_pi
+*		[in] Pointer to a PortInfo attribute.
+*
+* RETURN VALUES
+*	Returns the encoded value for the neighbor MTU at this port.
+*
+* NOTES
+*
+* SEE ALSO
+*********/
+/****f* IBA Base: Types/ib_port_info_set_sm_sl
+* NAME
+*	ib_port_info_set_sm_sl
+*
+* DESCRIPTION
+*	Sets the SM sl value in the PortInfo attribute.
+*
+* SYNOPSIS
+*/
+AL_INLINE void AL_API
+ib_port_info_set_sm_sl(
+	IN				ib_port_info_t* const		p_pi,
+	IN		const	uint8_t						sm_sl )
+{
+	CL_ASSERT( sm_sl<= 5 );
+	CL_ASSERT( sm_sl != 0 );
+	p_pi->mtu_smsl = (uint8_t)((p_pi->mtu_smsl & 0xF0) | sm_sl );
+}
+/*
+* PARAMETERS
+*	p_pi
+*		[in] Pointer to a PortInfo attribute.
+*
+*	mtu
+*		[in] Encoded SM sl value to set
+*
+* RETURN VALUES
+*	None.
+*
+* NOTES
+*
+* SEE ALSO
+*********/
+
 /****f* IBA Base: Types/ib_port_info_set_timeout
 * NAME
 *	ib_port_info_set_timeout
diff --git a/trunk/inc/user/iba/ib_uvp.h b/trunk/inc/user/iba/ib_uvp.h
index 93eae057..edc98154 100644
--- a/trunk/inc/user/iba/ib_uvp.h
+++ b/trunk/inc/user/iba/ib_uvp.h
@@ -131,7 +131,6 @@
 *	1. Register physical memory region with HCA (ci_register_pmr)
 *	2. Modify physical memory region with HCA (ci_modify_pmr)
 *	3. Create Special QP	(ci_create_spl_qp)
-*	4. Local Mad (ci_local_mad)
 *
 *	For all these functions, the vendor does NOT provide support
 *	and UAL will return IB_UNSUPPORTED to the caller of Access Layer.