From: shefty Date: Fri, 8 May 2009 18:34:41 +0000 (+0000) Subject: winverbs branch: fix MAD routing to support ibping -S option X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=a34cc3a02c199ca7e99c9fe7239e94013cf3a673;p=~shefty%2Frdma-win.git winverbs branch: fix MAD routing to support ibping -S option Signed-off-by: Sean Hefty git-svn-id: svn://openib.tc.cornell.edu/gen1@2157 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- diff --git a/branches/winverbs/core/al/al_mad.c b/branches/winverbs/core/al/al_mad.c index f60e8fbb..4576fa8a 100644 --- a/branches/winverbs/core/al/al_mad.c +++ b/branches/winverbs/core/al/al_mad.c @@ -1027,7 +1027,6 @@ __use_tid_routing( IN const ib_mad_t* const p_mad_hdr, IN const boolean_t are_we_sender ) { - ib_rmpp_mad_t *p_rmpp_mad; boolean_t is_orig; AL_ENTER( AL_DBG_MAD_SVC ); @@ -1039,45 +1038,10 @@ __use_tid_routing( return FALSE; } - /* - * Determine originator for a sent MAD. Received MADs are just the - * opposite. - */ - - /* Non-DATA RMPP MADs are handled differently. */ - p_rmpp_mad = (ib_rmpp_mad_t*)p_mad_hdr; - if( (p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_ADM) && - ( ib_rmpp_is_flag_set( p_rmpp_mad, IB_RMPP_FLAG_ACTIVE ) && - (p_rmpp_mad->rmpp_type != IB_RMPP_TYPE_DATA) ) ) - { - /* - * We need to distinguish between ACKs sent after receiving - * a request, versus ACKs sent after receiving a response. ACKs - * to a request are from the responder. ACKs to a response are - * from the originator. - - * Note that we assume STOP and ABORT packets are initiated by - * receivers. If both senders and receivers can - * initiate STOP and ABORT MADs, then we can't distinguish which - * transaction is associated with the MAD. The TID for a - * send and receive can be the same. - */ - is_orig = !ib_mad_is_response( p_mad_hdr ); - } + if (are_we_sender) + is_orig = !ib_mad_is_response(p_mad_hdr); else - { - /* - * See if the MAD is being sent in response to a previous MAD. If - * it is, then we're NOT the originator. Note that trap repress - * MADs are responses, even though the response bit isn't set. - */ - is_orig = !( ib_mad_is_response( p_mad_hdr ) || - (p_mad_hdr->method == IB_MAD_METHOD_TRAP_REPRESS) ); - } - - /* If we're the receiver, toggle the result. */ - if( !are_we_sender ) - is_orig = !is_orig; + is_orig = ib_mad_is_response(p_mad_hdr); AL_EXIT( AL_DBG_MAD_SVC ); return is_orig; @@ -1616,6 +1580,13 @@ __does_send_req_rmpp( return FALSE; } } + if (ib_class_is_vendor_specific_high(p_mad_element->p_mad_buf->mgmt_class) && + ib_rmpp_is_flag_set((ib_rmpp_mad_t *) p_mad_element->p_mad_buf, + IB_RMPP_FLAG_ACTIVE)) + { + *p_rmpp_version = DEFAULT_RMPP_VERSION; + return TRUE; + } /* The RMPP is not active. */ return FALSE; @@ -2221,12 +2192,9 @@ __mad_svc_recv_done( } /* - * See if the MAD was sent in response to a previously sent MAD. Note - * that trap repress messages are responses, even though the response - * bit isn't set. + * See if the MAD was sent in response to a previously sent MAD. */ - if( ib_mad_is_response( p_mad_hdr ) || - (p_mad_hdr->method == IB_MAD_METHOD_TRAP_REPRESS) ) + if( ib_mad_is_response( p_mad_hdr ) ) { /* Process the received response. */ __process_recv_resp( h_mad_svc, p_mad_element ); @@ -2336,9 +2304,9 @@ __recv_requires_rmpp( switch( mad_svc_type ) { case IB_MAD_SVC_DEFAULT: - /* Only subnet management receives require RMPP. */ - return( (p_rmpp_mad->common_hdr.mgmt_class == IB_MCLASS_SUBN_ADM) && - ib_rmpp_is_flag_set( p_rmpp_mad, IB_RMPP_FLAG_ACTIVE ) ); + return (p_rmpp_mad->common_hdr.mgmt_class == IB_MCLASS_SUBN_ADM || + ib_class_is_vendor_specific_high(p_rmpp_mad->common_hdr.mgmt_class)) && + ib_rmpp_is_flag_set(p_rmpp_mad, IB_RMPP_FLAG_ACTIVE); case IB_MAD_SVC_RMPP: return( ib_rmpp_is_flag_set( p_rmpp_mad, IB_RMPP_FLAG_ACTIVE ) ); @@ -2367,9 +2335,9 @@ __is_internal_send( { case IB_MAD_SVC_DEFAULT: /* Internal sends are non-RMPP data MADs. */ - return( (p_rmpp_mad->common_hdr.mgmt_class == IB_MCLASS_SUBN_ADM) && - (p_rmpp_mad->rmpp_type && - (p_rmpp_mad->rmpp_type != IB_RMPP_TYPE_DATA) ) ); + return ((p_rmpp_mad->common_hdr.mgmt_class == IB_MCLASS_SUBN_ADM || + ib_class_is_vendor_specific_high(p_rmpp_mad->common_hdr.mgmt_class)) && + (p_rmpp_mad->rmpp_type && p_rmpp_mad->rmpp_type != IB_RMPP_TYPE_DATA)); case IB_MAD_SVC_RMPP: /* The RMPP header is present. Check its type. */ diff --git a/branches/winverbs/core/al/kernel/al_smi.c b/branches/winverbs/core/al/kernel/al_smi.c index 012f96bc..7f5abee9 100644 --- a/branches/winverbs/core/al/kernel/al_smi.c +++ b/branches/winverbs/core/al/kernel/al_smi.c @@ -172,11 +172,11 @@ process_mad_recv( IN spl_qp_svc_t* p_spl_qp_svc, IN ib_mad_element_t* p_mad_element ); -mad_route_t +static mad_route_t route_recv_smp( IN ib_mad_element_t* p_mad_element ); -mad_route_t +static mad_route_t route_recv_smp_attr( IN ib_mad_element_t* p_mad_element ); @@ -184,12 +184,12 @@ mad_route_t route_recv_dm_mad( IN ib_mad_element_t* p_mad_element ); -mad_route_t -route_recv_gmp( +static mad_route_t +route_recv_bm( IN ib_mad_element_t* p_mad_element ); -mad_route_t -route_recv_gmp_attr( +static mad_route_t +route_recv_perf( IN ib_mad_element_t* p_mad_element ); ib_api_status_t @@ -2234,6 +2234,7 @@ fwd_local_mad( ib_mad_t* p_mad; ib_smp_t* p_smp; al_mad_send_t* p_mad_send; + ib_mad_element_t* p_send_mad; ib_mad_element_t* p_mad_response = NULL; ib_mad_t* p_mad_response_buf; ib_api_status_t status = IB_SUCCESS; @@ -2263,6 +2264,8 @@ fwd_local_mad( return status; } p_mad_response_buf = p_mad_response->p_mad_buf; + /* Copy MAD to dispatch locally in case CA doesn't handle it. */ + *p_mad_response_buf = *p_mad; } else { @@ -2399,9 +2402,17 @@ fwd_local_mad( /* Construct the receive MAD element. */ - p_mad_response->status = IB_WCS_SUCCESS; - p_mad_response->remote_qp = p_mad_wr->send_wr.dgrm.ud.remote_qp; - p_mad_response->remote_lid = p_spl_qp_svc->base_lid; + p_send_mad = p_mad_send->p_send_mad; + p_mad_response->status = IB_WCS_SUCCESS; + p_mad_response->grh_valid = p_send_mad->grh_valid; + if( p_mad_response->grh_valid ) + *p_mad_response->p_grh = *p_send_mad->p_grh; + p_mad_response->path_bits = p_send_mad->path_bits; + p_mad_response->pkey_index = p_send_mad->pkey_index; + p_mad_response->remote_lid = p_send_mad->remote_lid; + p_mad_response->remote_qkey = p_send_mad->remote_qkey; + p_mad_response->remote_qp = p_send_mad->remote_qp; + p_mad_response->remote_sl = p_send_mad->remote_sl; if( p_mad_wr->send_wr.send_opt & IB_RECV_OPT_IMMEDIATE ) { p_mad_response->immediate_data = p_mad_wr->send_wr.immediate_data; @@ -2413,6 +2424,8 @@ fwd_local_mad( * the send. This guarantees that the send request cannot time out. */ status = mad_disp_recv_done( p_spl_qp_svc->h_mad_disp, p_mad_response ); + if( status != IB_SUCCESS ) + ib_put_mad( p_mad_response ); } __complete_send_mad( p_spl_qp_svc->h_mad_disp, p_mad_wr,IB_WCS_SUCCESS); @@ -2937,35 +2950,19 @@ process_mad_recv( break; case IB_MCLASS_PERF: - /* Process the received GMP. */ - switch( p_mad_element->p_mad_buf->method ) - { - case IB_MAD_METHOD_GET: - case IB_MAD_METHOD_SET: - route = ROUTE_LOCAL; - break; - default: - break; - } + route = route_recv_perf( p_mad_element ); break; case IB_MCLASS_BM: - route = route_recv_gmp( p_mad_element ); + route = route_recv_bm( p_mad_element ); break; - case IB_MCLASS_SUBN_ADM: - case IB_MCLASS_DEV_MGMT: - case IB_MCLASS_COMM_MGMT: - case IB_MCLASS_SNMP: + case IB_MLX_VENDOR_CLASS1: + case IB_MLX_VENDOR_CLASS2: + route = ROUTE_LOCAL; break; default: - /* Route vendor specific MADs to the HCA provider. */ - if( ib_class_is_vendor_specific( - p_mad_element->p_mad_buf->mgmt_class ) ) - { - route = route_recv_gmp( p_mad_element ); - } break; } } @@ -2989,7 +2986,7 @@ process_mad_recv( /* * Route a received SMP. */ -mad_route_t +static mad_route_t route_recv_smp( IN ib_mad_element_t* p_mad_element ) { @@ -3054,7 +3051,7 @@ route_recv_smp( /* * Route received SMP attributes. */ -mad_route_t +static mad_route_t route_recv_smp_attr( IN ib_mad_element_t* p_mad_element ) { @@ -3090,72 +3087,38 @@ route_recv_smp_attr( } -/* - * Route a received GMP. - */ -mad_route_t -route_recv_gmp( +static mad_route_t +route_recv_bm( IN ib_mad_element_t* p_mad_element ) { - mad_route_t route; - - AL_ENTER( AL_DBG_SMI ); - - CL_ASSERT( p_mad_element ); - - /* Process the received GMP. */ switch( p_mad_element->p_mad_buf->method ) { case IB_MAD_METHOD_GET: case IB_MAD_METHOD_SET: - /* Route vendor specific MADs to the HCA provider. */ - if( ib_class_is_vendor_specific( - p_mad_element->p_mad_buf->mgmt_class ) ) - { - route = ROUTE_LOCAL; - } - else - { - route = route_recv_gmp_attr( p_mad_element ); - } + if( p_mad_element->p_mad_buf->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO ) + return ROUTE_LOCAL; break; - default: - route = ROUTE_DISPATCHER; break; } - - AL_EXIT( AL_DBG_SMI ); - return route; + return ROUTE_DISPATCHER; } - - -/* - * Route received GMP attributes. - */ -mad_route_t -route_recv_gmp_attr( +static mad_route_t +route_recv_perf( IN ib_mad_element_t* p_mad_element ) { - mad_route_t route; - - AL_ENTER( AL_DBG_SMI ); - - CL_ASSERT( p_mad_element ); - - /* Process the received GMP attributes. */ - if( p_mad_element->p_mad_buf->attr_id == IB_MAD_ATTR_CLASS_PORT_INFO ) - route = ROUTE_LOCAL; - else - route = ROUTE_DISPATCHER; - - AL_EXIT( AL_DBG_SMI ); - return route; + switch( p_mad_element->p_mad_buf->method ) + { + case IB_MAD_METHOD_GET: + case IB_MAD_METHOD_SET: + return ROUTE_LOCAL; + default: + break; + } + return ROUTE_DISPATCHER; } - - /* * Forward a locally generated Subnet Management trap. */ @@ -3263,8 +3226,7 @@ recv_local_mad( * We need to get a response from the local HCA to this MAD only if this * MAD is not itself a response. */ - p_mad_request->resp_expected = !( ib_mad_is_response( p_mad_hdr ) || - ( p_mad_hdr->method == IB_MAD_METHOD_TRAP_REPRESS ) ); + p_mad_request->resp_expected = !ib_mad_is_response( p_mad_hdr ); p_mad_request->timeout_ms = LOCAL_MAD_TIMEOUT; p_mad_request->send_opt = IB_SEND_OPT_LOCAL; diff --git a/branches/winverbs/core/winmad/kernel/wm_provider.c b/branches/winverbs/core/winmad/kernel/wm_provider.c index 56df62fc..a150bf1e 100644 --- a/branches/winverbs/core/winmad/kernel/wm_provider.c +++ b/branches/winverbs/core/winmad/kernel/wm_provider.c @@ -106,7 +106,7 @@ void WmProviderDeregister(WM_PROVIDER *pProvider, WM_REGISTRATION *pRegistration next = mad->p_next; mad->p_next = NULL; - if (mad->send_context1 == pRegistration) { + if (mad->context1 == pRegistration) { pRegistration->pDevice->IbInterface.put_mad(mad); } else { WmInsertMad(pProvider, mad); @@ -215,14 +215,9 @@ void WmProviderRemoveHandler(WM_PROVIDER *pProvider, WM_IB_DEVICE *pDevice) WmProviderUnlockRemove(pProvider); } -static NTSTATUS WmCopyRead(WM_PROVIDER *pProvider, WM_IO_MAD *pIoMad, - ib_mad_element_t *pMad, size_t *pLen) +static NTSTATUS WmCopyMad(WM_IO_MAD *pIoMad, ib_mad_element_t *pMad, size_t *pLen) { - WM_REGISTRATION *reg; - - reg = (WM_REGISTRATION *) pMad->send_context1; - pIoMad->Id = reg->Id; - + pIoMad->Id = ((WM_REGISTRATION *) pMad->context1)->Id; pIoMad->Status = pMad->status; pIoMad->Timeout = pMad->timeout_ms; pIoMad->Retries = pMad->retry_cnt; @@ -278,9 +273,9 @@ void WmProviderRead(WM_PROVIDER *pProvider, WDFREQUEST Request) } len = outlen; - status = WmCopyRead(pProvider, wmad, pProvider->MadHead, &len); + status = WmCopyMad(wmad, pProvider->MadHead, &len); if (status == STATUS_SUCCESS) { - reg = (WM_REGISTRATION *) pProvider->MadHead->send_context1; + reg = (WM_REGISTRATION *) pProvider->MadHead->context1; reg->pDevice->IbInterface.put_mad(WmRemoveMad(pProvider)); } WdfObjectReleaseLock(pProvider->ReadQueue); @@ -302,7 +297,6 @@ static NTSTATUS WmSendMad(WM_REGISTRATION *pRegistration, WM_IO_MAD *pIoMad, UIN return STATUS_NO_MEMORY; } - mad->context1 = pRegistration; RtlCopyMemory(mad->p_mad_buf, pIoMad + 1, size); mad->remote_qp = pIoMad->Address.Qpn; mad->remote_qkey = pIoMad->Address.Qkey; @@ -322,7 +316,9 @@ static NTSTATUS WmSendMad(WM_REGISTRATION *pRegistration, WM_IO_MAD *pIoMad, UIN mad->remote_sl = pIoMad->Address.ServiceLevel; mad->pkey_index = pIoMad->Address.PkeyIndex; mad->path_bits = pIoMad->Address.PathBits; - mad->p_mad_buf->trans_id &= 0xFFFFFFFF00000000; + if (!ib_mad_is_response(mad->p_mad_buf)) { + mad->p_mad_buf->trans_id &= 0xFFFFFFFF00000000; + } ib_status = pifc->send_mad(pRegistration->hService, mad, NULL); if (ib_status != IB_SUCCESS) { @@ -366,15 +362,16 @@ out: void WmReceiveHandler(ib_mad_svc_handle_t hService, void *Context, ib_mad_element_t *pMad) { - WM_REGISTRATION *reg; - WM_PROVIDER *prov = Context; + WM_REGISTRATION *reg = Context; + WM_PROVIDER *prov; WDFREQUEST request; NTSTATUS status; WM_IO_MAD *wmad; size_t len = 0; UNREFERENCED_PARAMETER(hService); - reg = (WM_REGISTRATION *) pMad->send_context1; + prov = reg->pProvider; + pMad->context1 = reg; WdfObjectAcquireLock(prov->ReadQueue); if (reg->hService == NULL) { @@ -394,9 +391,8 @@ void WmReceiveHandler(ib_mad_svc_handle_t hService, void *Context, goto complete; } - status = WmCopyRead(prov, wmad, pMad, &len); + status = WmCopyMad(wmad, pMad, &len); if (status == STATUS_SUCCESS) { - reg = (WM_REGISTRATION *) pMad->send_context1; reg->pDevice->IbInterface.put_mad(pMad); } else { WmInsertMad(prov, pMad); @@ -412,9 +408,8 @@ void WmSendHandler(ib_mad_svc_handle_t hService, void *Context, ib_mad_element_t *pMad) { if (pMad->status == IB_SUCCESS) { - ((WM_REGISTRATION *) pMad->context1)->pDevice->IbInterface.put_mad(pMad); + ((WM_REGISTRATION *) Context)->pDevice->IbInterface.put_mad(pMad); } else { - pMad->send_context1 = (void*) pMad->context1; WmReceiveHandler(hService, Context, pMad); } } diff --git a/branches/winverbs/core/winmad/kernel/wm_reg.c b/branches/winverbs/core/winmad/kernel/wm_reg.c index 22c02114..59f50556 100644 --- a/branches/winverbs/core/winmad/kernel/wm_reg.c +++ b/branches/winverbs/core/winmad/kernel/wm_reg.c @@ -146,7 +146,7 @@ static NTSTATUS WmRegInit(WM_REGISTRATION *pRegistration, WM_IO_REGISTER *pAttri goto err4; } - svc.mad_svc_context = pRegistration->pProvider; + svc.mad_svc_context = pRegistration; svc.pfn_mad_send_cb = WmSendHandler; svc.pfn_mad_recv_cb = WmReceiveHandler; svc.support_unsol = WmConvertMethods(&svc, pAttributes); diff --git a/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c b/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c index 4a055d95..ed1de54c 100644 --- a/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c +++ b/branches/winverbs/hw/mlx4/kernel/bus/ib/mad.c @@ -44,11 +44,6 @@ #endif -enum { - MLX4_IB_VENDOR_CLASS1 = 0x9, - MLX4_IB_VENDOR_CLASS2 = 0xa -}; - int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int ignore_mkey, int ignore_bkey, int port, ib_wc_t *in_wc, struct ib_grh *in_grh, void *in_mad, void *response_mad) @@ -240,8 +235,8 @@ int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, IB_SMP_ATTR_VENDOR_MASK)) return IB_MAD_RESULT_SUCCESS; } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || - in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS1 || - in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS2) { + in_mad->mad_hdr.mgmt_class == IB_MLX_VENDOR_CLASS1 || + in_mad->mad_hdr.mgmt_class == IB_MLX_VENDOR_CLASS2) { if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) return IB_MAD_RESULT_SUCCESS; diff --git a/branches/winverbs/hw/mthca/kernel/mthca_mad.c b/branches/winverbs/hw/mthca/kernel/mthca_mad.c index f001805d..6c6c0e6f 100644 --- a/branches/winverbs/hw/mthca/kernel/mthca_mad.c +++ b/branches/winverbs/hw/mthca/kernel/mthca_mad.c @@ -45,11 +45,6 @@ #endif #include "mthca_cmd.h" -enum { - MTHCA_VENDOR_CLASS1 = 0x9, - MTHCA_VENDOR_CLASS2 = 0xa -}; - struct mthca_trap_mad { struct scatterlist sg; }; @@ -231,8 +226,8 @@ int mthca_process_mad(struct ib_device *ibdev, } else { if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || - in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS1 || - in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS2) { + in_mad->mad_hdr.mgmt_class == IB_MLX_VENDOR_CLASS1 || + in_mad->mad_hdr.mgmt_class == IB_MLX_VENDOR_CLASS2) { if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) { diff --git a/branches/winverbs/inc/iba/ib_types.h b/branches/winverbs/inc/iba/ib_types.h index aaa497b1..f1d88ef5 100644 --- a/branches/winverbs/inc/iba/ib_types.h +++ b/branches/winverbs/inc/iba/ib_types.h @@ -534,6 +534,9 @@ extern "C" #define IB_MCLASS_VENDOR_LOW_RANGE_MAX 0x0f /**********/ +#define IB_MLX_VENDOR_CLASS1 0x9 +#define IB_MLX_VENDOR_CLASS2 0xA + /****d* IBA Base: Constants/IB_MCLASS_DEV_ADM * NAME * IB_MCLASS_DEV_ADM @@ -3795,8 +3798,8 @@ ib_mad_is_response( IN const ib_mad_t* const p_mad ) { CL_ASSERT( p_mad ); - return( (p_mad->method & IB_MAD_METHOD_RESP_MASK) == - IB_MAD_METHOD_RESP_MASK ); + return ((p_mad->method & IB_MAD_METHOD_RESP_MASK) || + (p_mad->method == IB_MAD_METHOD_TRAP_REPRESS)); } /* * PARAMETERS