From d88484e0df4d4e803f56a5f2d0dc482ea988e67f Mon Sep 17 00:00:00 2001 From: aestrin Date: Thu, 29 Jan 2009 19:39:06 +0000 Subject: [PATCH] [ipoib cm] revoked previous changes from revisions 1873, 1885. One preallocated send descriptor is all that needed for large packets segmentation. git-svn-id: svn://openib.tc.cornell.edu/gen1@1918 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- branches/ipoib_cm/kernel/ipoib_cm.c | 5 +- branches/ipoib_cm/kernel/ipoib_log.mc | 7 - branches/ipoib_cm/kernel/ipoib_port.c | 195 ++------------------------ branches/ipoib_cm/kernel/ipoib_port.h | 22 +-- 4 files changed, 15 insertions(+), 214 deletions(-) diff --git a/branches/ipoib_cm/kernel/ipoib_cm.c b/branches/ipoib_cm/kernel/ipoib_cm.c index 09eac11a..f7afc311 100644 --- a/branches/ipoib_cm/kernel/ipoib_cm.c +++ b/branches/ipoib_cm/kernel/ipoib_cm.c @@ -796,7 +796,6 @@ __endpt_cm_send_cb( uint32_t total_length = 0; ipoib_endpt_t *p_endpt; send_buf_t *p_send_buf; - ipoib_send_desc_t *p_desc; size_t i; BOOLEAN send_failed = FALSE; @@ -832,9 +831,7 @@ __endpt_cm_send_cb( CL_ASSERT( p_packet ); CL_ASSERT( IPOIB_PORT_FROM_PACKET( p_packet ) == p_port ); - p_desc = IPOIB_SEND_FROM_PACKET( p_packet ); - p_send_buf = p_desc->p_buf; - ipoib_send_desc_mgr_put( p_port, p_desc ); + p_send_buf = IPOIB_SEND_FROM_PACKET( p_packet ); switch( p_wc->status ) { diff --git a/branches/ipoib_cm/kernel/ipoib_log.mc b/branches/ipoib_cm/kernel/ipoib_log.mc index 4a6310b6..bb7d1c1d 100644 --- a/branches/ipoib_cm/kernel/ipoib_log.mc +++ b/branches/ipoib_cm/kernel/ipoib_log.mc @@ -332,10 +332,3 @@ Language=English %2: Connected Mode initialized and operational. . -MessageId=0x005E -Facility=IPoIB -Severity=Error -SymbolicName=EVENT_IPOIB_SEND_DESC_POOL -Language=English -%2: Failed to create send descriptors pool. -. diff --git a/branches/ipoib_cm/kernel/ipoib_port.c b/branches/ipoib_cm/kernel/ipoib_port.c index d6d562a9..4e3bba24 100644 --- a/branches/ipoib_cm/kernel/ipoib_port.c +++ b/branches/ipoib_cm/kernel/ipoib_port.c @@ -380,24 +380,6 @@ IN uint8_t* p_buf, IN uint8_t* p_options, IN uint32_t options_len, IN BOOLEAN copy_all ); - -/***************************************************************************** - Send Descriptors management -*****************************************************************************/ -static ib_api_status_t -__send_desc_mgr_init( - IN ipoib_port_t* const p_port ); - -static void -__send_desc_mgr_destroy( - IN ipoib_port_t* const p_port ); - -static cl_status_t -__send_desc_ctor( - IN void* const p_object, - IN void* context, - OUT cl_pool_item_t** const pp_pool_item ); - /****************************************************************************** * * Endpoint manager operations @@ -689,16 +671,6 @@ __port_init( ("cl_spinlock_init returned %#x\n", cl_status) ); return IB_ERROR; } - - /* initialize send descriptors manager. */ - status = __send_desc_mgr_init( p_port ); - if( status != IB_SUCCESS ) - { - IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR, - ("__send_desc_mgr_init returned %s\n", - p_adapter->p_ifc->get_err_str( status )) ); - return status; - } /* Initialize the IB resource manager. */ status = __ib_mgr_init( p_port ); @@ -829,8 +801,7 @@ __port_cleanup( /* Wait for all sends and receives to get flushed. */ while( p_port->send_mgr.depth || p_port->recv_mgr.depth ) cl_thread_suspend( 0 ); - - __send_desc_mgr_destroy( p_port ); + /* Destroy the send and receive managers before closing the CA. */ __ib_mgr_destroy( p_port ); @@ -4181,8 +4152,7 @@ __send_mgr_filter_arp( { IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT, (" ARP %s SEND to ENDPT[%p] State: %d flag: %#x, QPN: %#x MAC %02x:%02x:%02x:%02x:%02x:%02x\n", - ( p_ib_arp->op == ARP_OP_REP ? "REP": "REQ"), - p_desc->p_endpt, + ( p_ib_arp->op == ARP_OP_REP ? "REP": "REQ"), p_desc->p_endpt, endpt_cm_get_state( p_desc->p_endpt ), p_desc->p_endpt->cm_flag, cl_ntoh32( ipoib_addr_get_qpn( &p_ib_arp->dst_hw )), @@ -4421,7 +4391,7 @@ __build_send_desc( /* Store context in our reserved area of the packet. */ IPOIB_PORT_FROM_PACKET( p_desc->p_pkt ) = p_port; IPOIB_ENDPT_FROM_PACKET( p_desc->p_pkt ) = p_desc->p_endpt; - IPOIB_SEND_FROM_PACKET( p_desc->p_pkt ) = p_desc; + IPOIB_SEND_FROM_PACKET( p_desc->p_pkt ) = p_desc->p_buf; IPOIB_EXIT( IPOIB_DBG_SEND ); return NDIS_STATUS_SUCCESS; @@ -4525,7 +4495,7 @@ __process_failed_send( ExFreeToNPagedLookasideList( &p_port->buf_mgr.send_buf_list, p_desc->p_buf ); } - ipoib_send_desc_mgr_put( p_port, p_desc ); + IPOIB_EXIT( IPOIB_DBG_SEND ); } @@ -4578,24 +4548,13 @@ ipoib_port_send( cl_spinlock_acquire( &p_port->send_lock ); for( i = 0; i < num_packets; i++ ) { - p_desc = ipoib_send_desc_mgr_get( p_port ); - if( !p_desc ) - { - IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND, - ("No available Send Descriptors.\n") ); - while( i < num_packets ) - { - cl_qlist_insert_tail( &p_port->send_mgr.pending_list, - IPOIB_LIST_ITEM_FROM_PACKET( p_packet_array[i++] ) ); - } - break; - } - + p_desc = &p_port->send_mgr.desc; p_desc->p_pkt = p_packet_array[i]; p_desc->p_endpt = NULL; p_desc->p_buf = NULL; p_desc->send_qp = NULL; p_desc->num_wrs = 1; + p_desc->send_dir = 0; /* Get the ethernet header so we can find the endpoint. */ cl_perf_start( GetEthHdr ); @@ -4661,7 +4620,6 @@ h_end: cl_perf_stop( &p_port->p_adapter->perf, SendMgrQueue ); if( status == NDIS_STATUS_PENDING ) { - ipoib_send_desc_mgr_put( p_port, p_desc ); /* Queue all remaining packets. */ cl_perf_start( QueuePacket ); while( i < num_packets ) @@ -4693,7 +4651,6 @@ h_end: if( status == NDIS_STATUS_PENDING ) { ipoib_endpt_deref( p_desc->p_endpt ); - ipoib_send_desc_mgr_put( p_port, p_desc ); break; } cl_perf_start( ProcessFailedSends ); @@ -4771,19 +4728,14 @@ ipoib_port_resume( ("No available WQEs.\n") ); break; } - p_desc = ipoib_send_desc_mgr_get( p_port ); - if( !p_desc ) - { - IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND, - ("No available Send Descriptors.\n") ); - break; - } + p_desc = &p_port->send_mgr.desc; p_desc->p_pkt = IPOIB_PACKET_FROM_LIST_ITEM( cl_qlist_remove_head( &p_port->send_mgr.pending_list ) ); p_desc->p_endpt = NULL; p_desc->p_buf = NULL; p_desc->send_qp = NULL; p_desc->num_wrs = 1; + p_desc->send_dir = 0; /* Get the ethernet header so we can find the endpoint. */ status = __send_mgr_get_eth_hdr( @@ -4804,7 +4756,6 @@ ipoib_port_resume( CL_ASSERT( p_desc->p_endpt == NULL ); cl_qlist_insert_head( &p_port->send_mgr.pending_list, IPOIB_LIST_ITEM_FROM_PACKET( p_desc->p_pkt ) ); - ipoib_send_desc_mgr_put( p_port, p_desc ); break; } else if( status != NDIS_STATUS_SUCCESS ) @@ -4821,7 +4772,6 @@ ipoib_port_resume( ("Multicast Mac - trying to join.\n") ); cl_qlist_insert_head( &p_port->send_mgr.pending_list, IPOIB_LIST_ITEM_FROM_PACKET( p_desc->p_pkt ) ); - ipoib_send_desc_mgr_put( p_port, p_desc ); break; } } @@ -4845,7 +4795,6 @@ ipoib_port_resume( { /* ARP REPLY packet queued */ ipoib_endpt_deref( p_desc->p_endpt ); - ipoib_send_desc_mgr_put( p_port, p_desc ); break; } cl_perf_start( ProcessFailedSends ); @@ -4891,7 +4840,6 @@ __send_cb( uint32_t length; ipoib_endpt_t *p_endpt; send_buf_t *p_send_buf = NULL; - ipoib_send_desc_t *p_desc; ip_stat_sel_t type; size_t i; PERF_DECLARE( SendCompBundle ); @@ -4941,9 +4889,8 @@ __send_cb( CL_ASSERT( p_packet ); CL_ASSERT( IPOIB_PORT_FROM_PACKET( p_packet ) == p_port ); p_endpt = IPOIB_ENDPT_FROM_PACKET( p_packet ); - p_desc = IPOIB_SEND_FROM_PACKET( p_packet ); - p_send_buf = p_desc->p_buf; - ipoib_send_desc_mgr_put( p_port, p_desc ); + p_send_buf = IPOIB_SEND_FROM_PACKET( p_packet ); + if( p_endpt->h_mcast ) { if( p_endpt->dgid.multicast.raw_group_id[11] == 0xFF && @@ -4977,9 +4924,7 @@ __send_cb( if( p_packet ) { p_endpt = IPOIB_ENDPT_FROM_PACKET( p_packet ); - p_desc = IPOIB_SEND_FROM_PACKET( p_packet ); - p_send_buf = p_desc->p_buf; - ipoib_send_desc_mgr_put( p_port, p_desc ); + p_send_buf = IPOIB_SEND_FROM_PACKET( p_packet ); ipoib_inc_send_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 ); NdisMSendCompleteX( p_port->p_adapter->h_adapter, p_packet, NDIS_STATUS_RESET_IN_PROGRESS ); @@ -4996,9 +4941,7 @@ __send_cb( if( p_packet ) { p_endpt = IPOIB_ENDPT_FROM_PACKET( p_packet ); - p_desc = IPOIB_SEND_FROM_PACKET( p_packet ); - p_send_buf = p_desc->p_buf; - ipoib_send_desc_mgr_put( p_port, p_desc ); + p_send_buf = IPOIB_SEND_FROM_PACKET( p_packet ); ipoib_inc_send_stat( p_port->p_adapter, IP_STAT_ERROR, 0 ); NdisMSendCompleteX( p_port->p_adapter->h_adapter, p_packet, NDIS_STATUS_FAILURE ); @@ -7570,117 +7513,3 @@ IN BOOLEAN copy_all ) } return; } - - -/*************************************************** - Send desriptors pool management -**************************************************/ - -static ib_api_status_t -__send_desc_mgr_init( - IN ipoib_port_t* const p_port ) -{ - cl_status_t cl_status; - - IPOIB_ENTER( IPOIB_DBG_INIT ); - - cl_qpool_construct( &p_port->send_mgr.desc_pool ); - cl_spinlock_init( &p_port->send_mgr.desc_lock ); - - /* Allocate send descriptors pool */ - cl_status = cl_qpool_init( &p_port->send_mgr.desc_pool, - p_port->p_adapter->params.sq_depth, - 0, - 10, - sizeof( ipoib_send_desc_t ), - __send_desc_ctor, - NULL, - p_port ); - - if( cl_status != CL_SUCCESS ) - { - NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter, - EVENT_IPOIB_SEND_DESC_POOL, 1, cl_status ); - - IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR, - ("cl_qpool_init for send descriptors pool returned %#x\n", cl_status) ); - - return IB_INSUFFICIENT_MEMORY; - } - - IPOIB_EXIT( IPOIB_DBG_INIT ); - return IB_SUCCESS; -} - -static void -__send_desc_mgr_destroy( - IN ipoib_port_t* const p_port ) -{ - - IPOIB_ENTER(IPOIB_DBG_INIT ); - - CL_ASSERT( p_port ); - cl_spinlock_destroy( &p_port->send_mgr.desc_lock ); - if( cl_is_qpool_inited( &p_port->send_mgr.desc_pool ) ) - { - cl_qpool_destroy( &p_port->send_mgr.desc_pool ); - } - IPOIB_EXIT( IPOIB_DBG_INIT ); -} - -static cl_status_t -__send_desc_ctor( - IN void* const p_object, - IN void* context, - OUT cl_pool_item_t** const pp_pool_item ) -{ - ipoib_send_desc_t* p_desc; - ipoib_port_t* p_port; - - CL_ASSERT( p_object ); - CL_ASSERT( context ); - - p_desc = (ipoib_send_desc_t*)p_object; - p_port = (ipoib_port_t*)context; - p_desc->p_endpt = NULL; - p_desc->p_buf = NULL; - p_desc->send_qp = NULL; - p_desc->num_wrs = 1; - - *pp_pool_item = &p_desc->pool_item; - return CL_SUCCESS; -} - -inline ipoib_send_desc_t* -ipoib_send_desc_mgr_get( - IN ipoib_port_t* const p_port ) -{ - ipoib_send_desc_t *p_desc; - cl_spinlock_acquire( &p_port->send_mgr.desc_lock ); - p_desc = (ipoib_send_desc_t*)cl_qpool_get( &p_port->send_mgr.desc_pool ); - cl_spinlock_release( &p_port->send_mgr.desc_lock ); - - return p_desc; -} - -void -ipoib_send_desc_mgr_put( - IN ipoib_port_t* const p_port, - IN ipoib_send_desc_t* const p_desc ) -{ - /* Return the descriptor to it's pool. */ - cl_spinlock_acquire( &p_port->send_mgr.desc_lock ); - cl_qpool_put( &p_port->send_mgr.desc_pool, &p_desc->pool_item ); - cl_spinlock_release( &p_port->send_mgr.desc_lock ); - -} - -void -ipoib_send_desc_mgr_put_list( - IN ipoib_port_t* const p_port, - IN cl_qlist_t* const p_list ) -{ - cl_spinlock_acquire( &p_port->send_mgr.desc_lock ); - cl_qpool_put_list( &p_port->send_mgr.desc_pool, p_list ); - cl_spinlock_release( &p_port->send_mgr.desc_lock ); -} diff --git a/branches/ipoib_cm/kernel/ipoib_port.h b/branches/ipoib_cm/kernel/ipoib_port.h index 38480bf5..316622a1 100644 --- a/branches/ipoib_cm/kernel/ipoib_port.h +++ b/branches/ipoib_cm/kernel/ipoib_port.h @@ -72,7 +72,7 @@ #define IPOIB_RECV_FROM_PACKET( P ) \ (((void **)P->MiniportReservedEx)[1]) #define IPOIB_SEND_FROM_PACKET( P ) \ - (((void **)P->MiniportReservedEx)[2]) + (((send_buf_t**)P->MiniportReservedEx)[2]) #define IPOIB_PACKET_FROM_LIST_ITEM( I ) \ (PARENT_STRUCT( I, NDIS_PACKET, MiniportReservedEx )) #define IPOIB_LIST_ITEM_FROM_PACKET( P ) \ @@ -401,7 +401,6 @@ typedef enum __send_dir typedef struct _ipoib_send_desc { - cl_pool_item_t pool_item; NDIS_PACKET *p_pkt; ipoib_endpt_t *p_endpt; send_buf_t *p_buf; @@ -470,9 +469,7 @@ typedef struct _ipoib_send_mgr { atomic32_t depth; cl_qlist_t pending_list; - cl_spinlock_t desc_lock; - cl_qpool_t desc_pool; - + ipoib_send_desc_t desc; } ipoib_send_mgr_t; /* @@ -796,19 +793,4 @@ __port_attr_to_mtu_size(uint32_t value) return 256; } } - -inline ipoib_send_desc_t* -ipoib_send_desc_mgr_get( - IN ipoib_port_t* const p_port ); - -void -ipoib_send_desc_mgr_put( - IN ipoib_port_t* const p_port, - IN ipoib_send_desc_t* const p_desc ); - -void -ipoib_send_desc_mgr_put_list( - IN ipoib_port_t* const p_port, - IN cl_qlist_t* const p_list ); - #endif /* _IPOIB_PORT_H_ */ -- 2.41.0