[IPoIB] Avoid the SM (1/4)

This patch removes the path query from IPoIB when trying to send a unicast packet.  Instead it uses fields form the broadcast group and the work completion to form an address vector.  This speeds ARP responses.

The limitation of this patch is that the rate for the address vector will be that of the broadcast group, which may be suboptimal in mixed SDR/DDR/QDR fabrics.

Signed-off-by: Fab Tillier <ftillier@microsoft.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1@1610 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_endpoint.c b/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
index d63e700045d746093f7050db43ff75b2123ea1f2..009e34f15a9f3cc0ab2c8be2478e02d22f54ecd9 100644 (file)
--- a/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
+++ b/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_endpoint.c
@@ -183,6 +183,7 @@ ipoib_endpt_set_mcast(
                return status;\r
        }\r
        p_endpt->h_mcast = p_mcast_rec->h_mcast;\r
+       p_endpt->dlid = p_mcast_rec->p_member_rec->mlid;\r
 \r
        IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
        return IB_SUCCESS;\r
@@ -279,9 +280,8 @@ ipoib_endpt_queue(
 {\r
        ib_api_status_t status;\r
        ipoib_port_t    *p_port;\r
-       ib_query_req_t  query;\r
-       ib_user_query_t info;\r
-       ib_path_rec_t   path;\r
+       ib_av_attr_t    av_attr;\r
+       net32_t                 flow_lbl;\r
 \r
        IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
 \r
@@ -291,108 +291,34 @@ ipoib_endpt_queue(
                return NDIS_STATUS_SUCCESS;\r
        }\r
 \r
-       if( p_endpt->h_query ||\r
-               p_endpt->qpn == CL_HTON32(0x00FFFFFF) )\r
+       if( p_endpt->qpn == CL_HTON32(0x00FFFFFF) )\r
        {\r
+               /*\r
+                * Handle a race between the mcast callback and a receive/send.  The QP\r
+                * is joined to the MC group before the MC callback is received, so it\r
+                * can receive packets, and NDIS can try to respond.  We need to delay\r
+                * a response until the MC callback runs and sets the AV.\r
+                */\r
                ipoib_endpt_deref( p_endpt );\r
                IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
                return NDIS_STATUS_PENDING;\r
        }\r
 \r
-       /* This is the first packet for this endpoint.  Query the SA. */\r
+       /* This is the first packet for this endpoint.  Create the AV. */\r
        p_port = __endpt_parent( p_endpt );\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
-\r
-       info.method = IB_MAD_METHOD_GETTABLE;\r
-       info.attr_id = IB_MAD_ATTR_PATH_RECORD;\r
-       info.attr_size = sizeof(ib_path_rec_t);\r
-       info.comp_mask = IB_PR_COMPMASK_DGID | IB_PR_COMPMASK_SGID |\r
-               IB_PR_COMPMASK_REVERSIBLE | IB_PR_COMPMASK_NUM_PATH;\r
-       info.p_attr = &path;\r
-\r
-       cl_memclr( &path, sizeof(ib_path_rec_t) );\r
-       path.dgid = p_endpt->dgid;\r
-       ib_gid_set_default( &path.sgid, p_port->p_adapter->guids.port_guid.guid );\r
-       path.num_path = 0x1;\r
-\r
-       cl_memclr( &query, sizeof(ib_query_req_t) );\r
-       query.query_type = IB_QUERY_USER_DEFINED;\r
-       query.p_query_input = &info;\r
-       query.port_guid = p_port->p_adapter->guids.port_guid.guid;\r
-       query.timeout_ms = p_port->p_adapter->params.sa_timeout;\r
-       query.retry_cnt = p_port->p_adapter->params.sa_retry_cnt;\r
-\r
-       query.query_context = p_endpt;\r
-       query.pfn_query_cb = __path_query_cb;\r
-\r
-       status = p_port->p_adapter->p_ifc->query(\r
-               p_port->p_adapter->h_al, &query, &p_endpt->h_query );\r
-       if( status != IB_SUCCESS )\r
-       {\r
-               IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ENDPT,\r
-                       ("ib_query for path returned %s\n", \r
-                       p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               ipoib_endpt_deref( p_endpt );\r
-               /* Flag the adapter as hung. */\r
-               p_port->p_adapter->hung = TRUE;\r
-       }\r
-\r
-       IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
-       return NDIS_STATUS_PENDING;\r
-}\r
-\r
-\r
-static void\r
-__path_query_cb(\r
-       IN                              ib_query_rec_t                          *p_query_rec )\r
-{\r
-       ib_api_status_t status;\r
-       ipoib_endpt_t   *p_endpt;\r
-       ipoib_port_t    *p_port;\r
-       ib_av_attr_t    av_attr;\r
-       ib_path_rec_t   *p_path;\r
-       net32_t                 flow_lbl;\r
-\r
-       IPOIB_ENTER( IPOIB_DBG_ENDPT );\r
-\r
-       p_endpt = (ipoib_endpt_t*)p_query_rec->query_context;\r
-       p_port = __endpt_parent( p_endpt );\r
-\r
-       cl_obj_lock( &p_endpt->obj );\r
-       p_endpt->h_query = NULL;\r
-       if( p_endpt->obj.state == CL_DESTROYING )\r
-       {\r
-               cl_obj_unlock( &p_endpt->obj );\r
-               ipoib_endpt_deref( p_endpt );\r
-               if( p_query_rec->p_result_mad )\r
-                       p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );\r
-               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                       ("Endpoint destroying, aborting.\n") );\r
-               return;\r
-       }\r
-       cl_obj_unlock( &p_endpt->obj );\r
-\r
-       if( p_query_rec->status != IB_SUCCESS || !p_query_rec->result_cnt )\r
-       {\r
-               p_port->p_adapter->hung = TRUE;\r
-               ipoib_endpt_deref( p_endpt );\r
-               if( p_query_rec->p_result_mad )\r
-                       p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );\r
-               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
-                       ("Path query failed with %s\n",\r
-                       p_port->p_adapter->p_ifc->get_err_str( p_query_rec->status )) );\r
-               return;\r
-       }\r
-\r
-       p_path = ib_get_query_path_rec( p_query_rec->p_result_mad, 0 );\r
-\r
        cl_memclr( &av_attr, sizeof(ib_av_attr_t) );\r
 \r
        av_attr.port_num = p_port->port_num;\r
 \r
-       av_attr.sl = ib_path_rec_sl( p_path );\r
-       av_attr.dlid = p_path->dlid;\r
+       ib_member_get_sl_flow_hop(\r
+               p_port->ib_mgr.bcast_rec.sl_flow_hop,\r
+               &av_attr.sl,\r
+               &flow_lbl,\r
+               &av_attr.grh.hop_limit\r
+               );\r
+\r
+       av_attr.dlid = p_endpt->dlid;\r
 \r
        /*\r
         * We always send the GRH so that we preferably lookup endpoints\r
@@ -402,38 +328,17 @@ __path_query_cb(
         * for which there is no match, something that doesn't work when\r
         * using LIDs only.\r
         */\r
-       flow_lbl = ib_path_rec_flow_lbl( p_path );\r
        av_attr.grh_valid = TRUE;\r
        av_attr.grh.ver_class_flow = ib_grh_set_ver_class_flow(\r
-               6, p_path->tclass, flow_lbl );\r
+               6, p_port->ib_mgr.bcast_rec.tclass, flow_lbl );\r
        av_attr.grh.resv1 = 0;\r
        av_attr.grh.resv2 = 0;\r
-       av_attr.grh.hop_limit = ib_path_rec_hop_limit( p_path );\r
-       av_attr.grh.src_gid = p_path->sgid;\r
-       av_attr.grh.dest_gid = p_path->dgid;\r
-       \r
-       cl_obj_lock( &p_port->obj );\r
-       if( !p_endpt->dlid )\r
-       {\r
-               cl_map_item_t   *p_qitem;\r
+       ib_gid_set_default( &av_attr.grh.src_gid, p_port->p_adapter->guids.port_guid.guid );\r
+       av_attr.grh.dest_gid = p_endpt->dgid;\r
 \r
-               /* This is a subnet local endpoint that does not have its LID set. */\r
-               p_endpt->dlid = p_path->dlid;\r
-               /*\r
-                * Insert the item in the LID map so that locally routed unicast\r
-                * traffic will resolve it properly.\r
-                */\r
-               p_qitem = cl_qmap_insert( &p_port->endpt_mgr.lid_endpts,\r
-                       p_endpt->dlid, &p_endpt->lid_item );\r
-               CL_ASSERT( p_qitem == &p_endpt->lid_item );\r
-       }\r
-       cl_obj_unlock( &p_port->obj );\r
-       av_attr.static_rate = ib_path_rec_rate( p_path );\r
+       av_attr.static_rate = p_port->ib_mgr.bcast_rec.rate;\r
        av_attr.path_bits = 0;\r
 \r
-       /* Done with the path record.  Release the MAD. */\r
-       p_port->p_adapter->p_ifc->put_mad( p_query_rec->p_result_mad );\r
-\r
        /* Create the AV. */\r
        status = p_port->p_adapter->p_ifc->create_av(\r
                p_port->ib_mgr.h_pd, &av_attr, &p_endpt->h_av );\r
@@ -445,13 +350,9 @@ __path_query_cb(
                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                        ("ib_create_av failed with %s\n", \r
                        p_port->p_adapter->p_ifc->get_err_str( status )) );\r
-               return;\r
+               return NDIS_STATUS_FAILURE;\r
        }\r
 \r
-       /* Try to send all pending sends. */\r
-       ipoib_port_resume( p_port );\r
-\r
-       /* Release the reference taken for the SA query. */\r
-       ipoib_endpt_deref( p_endpt );\r
        IPOIB_EXIT( IPOIB_DBG_ENDPT );\r
+       return NDIS_STATUS_SUCCESS;\r
 }\r

diff --git a/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_port.c b/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_port.c
index e3be853c490e1e6add5d0187c3bad8d0da2b904a..6e1e56c3f2ffd0496cb06fc80467b995ca80dbdf 100644 (file)
--- a/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_port.c
+++ b/branches/WOF2-0/trunk/ulp/ipoib/kernel/ipoib_port.c
@@ -1843,7 +1843,7 @@ __recv_get_endpts(
 #else  /* IPOIB_INLINE_RECV */\r
                        *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
 #endif /* IPOIB_INLINE_RECV */\r
-                               0, p_wc->recv.ud.remote_qp );\r
+                               p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
                        if( !*pp_src )\r
                        {\r
                                IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
@@ -2453,13 +2453,10 @@ __recv_arp(
                        return status;\r
                }\r
                /*\r
-                * Create the endpoint.  Note that the LID is left blank and will be\r
-                * resolved by a path query as needed.  This is done because the\r
-                * remote LID/GID from the work completion may not be the original\r
-                * initiator.\r
+                * Create the endpoint.\r
                 */\r
                *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
-                       0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
+                       p_wc->recv.ud.remote_lid, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
 \r
                if( !*pp_src )\r
                {\r