From d9e771da16ec2b360a222ceccbbca5d088e20ee5 Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Tue, 3 Mar 2009 09:25:26 -0800 Subject: [PATCH] uDAPL common: add 64 bit counters for IA, EP, and EVD's. -DDAPL_COUNTERS to build-in counters for cma and scm providers. New extension calls in dat_ib_extensions.h for counters dat_print_counters, dat_query_counters Counters for operations, async errors, and data Update dtestx (-p) with print and query counter examples Signed-off-by: Arlin Davis --- dapl/common/dapl_debug.c | 202 ++++++++++++++++---- dapl/common/dapl_ep_connect.c | 2 +- dapl/common/dapl_ep_create.c | 2 +- dapl/common/dapl_ep_disconnect.c | 2 +- dapl/common/dapl_ep_free.c | 2 +- dapl/common/dapl_ep_get_status.c | 4 +- dapl/common/dapl_ep_post_rdma_read.c | 1 - dapl/common/dapl_ep_post_rdma_write.c | 1 - dapl/common/dapl_ep_post_recv.c | 1 - dapl/common/dapl_ep_post_send.c | 1 - dapl/common/dapl_ep_util.c | 15 ++ dapl/common/dapl_evd_connection_callb.c | 3 +- dapl/common/dapl_evd_cq_async_error_callb.c | 1 + dapl/common/dapl_evd_dequeue.c | 9 +- dapl/common/dapl_evd_dto_callb.c | 2 +- dapl/common/dapl_evd_free.c | 3 +- dapl/common/dapl_evd_qp_async_error_callb.c | 1 + dapl/common/dapl_evd_un_async_error_callb.c | 1 + dapl/common/dapl_evd_util.c | 15 ++ dapl/common/dapl_ia_util.c | 14 ++ dapl/common/dapl_lmr_free.c | 11 +- dapl/common/dapl_psp_create.c | 3 +- dapl/common/dapl_psp_create_any.c | 2 + dapl/common/dapl_psp_free.c | 2 + dapl/common/dapl_pz_create.c | 1 + dapl/common/dapl_pz_free.c | 2 + dapl/common/dapl_rmr_create.c | 2 +- dapl/common/dapl_rmr_free.c | 3 + dapl/common/dapl_rsp_create.c | 2 + dapl/common/dapl_rsp_free.c | 2 + dapl/common/dapl_srq_create.c | 2 + dapl/common/dapl_srq_free.c | 2 + dapl/common/dapl_timer_util.c | 2 - dapl/ibal/dapl_ibal_extensions.c | 2 - dapl/include/dapl.h | 10 + dapl/include/dapl_debug.h | 55 ++---- dapl/openib_cma/dapl_ib_dto.h | 62 +++++- dapl/openib_cma/dapl_ib_extensions.c | 35 +++- dapl/openib_cma/dapl_ib_util.c | 5 + dapl/openib_scm/dapl_ib_dto.h | 74 ++++++- dapl/openib_scm/dapl_ib_extensions.c | 35 +++- dapl/openib_scm/dapl_ib_util.c | 5 + dapl/udapl/dapl_evd_create.c | 3 +- dapl/udapl/dapl_evd_wait.c | 6 +- dapl/udapl/dapl_init.c | 2 - dapl/udapl/dapl_lmr_create.c | 2 + dapl/udapl/linux/dapl_osd.h | 9 +- dat/include/dat2/dat_ib_extensions.h | 133 ++++++++++++- test/dtest/dtestx.c | 88 ++++++++- 49 files changed, 706 insertions(+), 138 deletions(-) diff --git a/dapl/common/dapl_debug.c b/dapl/common/dapl_debug.c index e717591..7601fc8 100644 --- a/dapl/common/dapl_debug.c +++ b/dapl/common/dapl_debug.c @@ -25,7 +25,6 @@ * and/or other materials provided with the distribution. */ -#include "dapl_debug.h" #include "dapl.h" #if !defined(__KDAPL__) #include @@ -67,52 +66,177 @@ void dapl_internal_dbg_log ( DAPL_DBG_TYPE type, const char *fmt, ...) } } -#if defined(DAPL_COUNTERS) -int dapl_dbg_counters[DCNT_NUM_COUNTERS] = { 0 }; +#ifdef DAPL_COUNTERS /* - * The order of this list must match exactly with the #defines - * in dapl_debug.h + * The order of this list must match the DAT counter definitions */ -char *dapl_dbg_counter_names[] = { - "dapl_ep_create", - "dapl_ep_free", - "dapl_ep_connect", - "dapl_ep_disconnect", - "dapl_ep_post_send", - "dapl_ep_post_recv", - "dapl_ep_post_rdma_write", - "dapl_ep_post_rdma_read", - "dapl_evd_create", - "dapl_evd_free", - "dapl_evd_wait", - "dapl_evd_blocked", - "dapl_evd_completion_notify", - "dapl_evd_dto_callback", - "dapl_evd_connection_callback", - "dapl_evd_dequeue", - "dapl_evd_poll", - "dapl_evd_found", - "dapl_evd_not_found", - "dapls_timer_set", - "dapls_timer_cancel", - "dapl_extensions" +static char *ia_cntr_names[] = { + "DCNT_IA_PZ_CREATE", + "DCNT_IA_PZ_FREE", + "DCNT_IA_LMR_CREATE", + "DCNT_IA_LMR_FREE", + "DCNT_IA_RMR_CREATE", + "DCNT_IA_RMR_FREE", + "DCNT_IA_PSP_CREATE", + "DCNT_IA_PSP_CREATE_ANY", + "DCNT_IA_PSP_FREE", + "DCNT_IA_RSP_CREATE", + "DCNT_IA_RSP_FREE", + "DCNT_IA_EVD_CREATE", + "DCNT_IA_EVD_FREE", + "DCNT_IA_EP_CREATE", + "DCNT_IA_EP_FREE", + "DCNT_IA_SRQ_CREATE", + "DCNT_IA_SRQ_FREE", + "DCNT_IA_SP_CR", + "DCNT_IA_SP_CR_ACCEPTED", + "DCNT_IA_SP_CR_REJECTED", + "DCNT_IA_MEM_ALLOC", + "DCNT_IA_MEM_ALLOC_DATA", + "DCNT_IA_MEM_FREE", + "DCNT_IA_ASYNC_ERROR", + "DCNT_IA_ASYNC_QP_ERROR", + "DCNT_IA_ASYNC_CQ_ERROR" }; -void dapl_dump_cntr( int cntr ) +static char *ep_cntr_names[] = { + "DCNT_EP_CONNECT", + "DCNT_EP_DISCONNECT", + "DCNT_EP_POST_SEND", + "DCNT_EP_POST_SEND_DATA", + "DCNT_EP_POST_SEND_UD", + "DCNT_EP_POST_SEND_UD_DATA", + "DCNT_EP_POST_RECV", + "DCNT_EP_POST_RECV_DATA", + "DCNT_EP_POST_WRITE", + "DCNT_EP_POST_WRITE_DATA", + "DCNT_EP_POST_WRITE_IMM", + "DCNT_EP_POST_WRITE_IMM_DATA", + "DCNT_EP_POST_READ", + "DCNT_EP_POST_READ_DATA", + "DCNT_EP_POST_CMP_SWAP", + "DCNT_EP_POST_FETCH_ADD", + "DCNT_EP_RECV", + "DCNT_EP_RECV_DATA", + "DCNT_EP_RECV_UD", + "DCNT_EP_RECV_UD_DATA", + "DCNT_EP_RECV_IMM", + "DCNT_EP_RECV_IMM_DATA", + "DCNT_EP_RECV_RDMA_IMM", + "DCNT_EP_RECV_RDMA_IMM_DATA", +}; + +static char *evd_cntr_names[] = { + "DCNT_EVD_WAIT", + "DCNT_EVD_WAIT_BLOCKED", + "DCNT_EVD_WAIT_NOTIFY", + "DCNT_EVD_DEQUEUE", + "DCNT_EVD_DEQUEUE_FOUND", + "DCNT_EVD_DEQUEUE_NOT_FOUND", + "DCNT_EVD_DEQUEUE_POLL", + "DCNT_EVD_DEQUEUE_POLL_FOUND", + "DCNT_EVD_CONN_CALLBACK", + "DCNT_EVD_DTO_CALLBACK", +}; + +DAT_RETURN dapl_query_counter(DAT_HANDLE dh, + int counter, + void *p_cntrs_out, + int reset) { - int i; + int i, max; + DAT_UINT64 *p_cntrs; + DAT_HANDLE_TYPE type = 0; + + dat_get_handle_type(dh, &type); - for ( i = 0; i < DCNT_NUM_COUNTERS; i++ ) - { - if (( cntr == i ) || ( cntr == DCNT_ALL_COUNTERS )) - { - dapl_dbg_log ( DAPL_DBG_TYPE_CNTR, - "DAPL Counter: %s = %lu \n", - dapl_dbg_counter_names[i], - dapl_dbg_counters[i] ); + switch(type) { + case DAT_HANDLE_TYPE_IA: + max = DCNT_IA_ALL_COUNTERS; + p_cntrs = ((DAPL_IA*)dh)->cntrs; + break; + case DAT_HANDLE_TYPE_EP: + max = DCNT_EP_ALL_COUNTERS; + p_cntrs = ((DAPL_EP*)dh)->cntrs; + break; + case DAT_HANDLE_TYPE_EVD: + max = DCNT_EVD_ALL_COUNTERS; + p_cntrs = ((DAPL_EVD*)dh)->cntrs; + break; + default: + return DAT_INVALID_HANDLE; + } + + for (i=0; i < max; i++) { + if ((counter == i) || (counter == max)) { + ((DAT_UINT64*)p_cntrs_out)[i] = p_cntrs[i]; + if (reset) p_cntrs[i]=0; + } + } + return DAT_SUCCESS; +} + +char *dapl_query_counter_name(DAT_HANDLE dh, int counter) +{ + DAT_HANDLE_TYPE type = 0; + + dat_get_handle_type(dh, &type); + + switch(type) { + case DAT_HANDLE_TYPE_IA: + if (counter < DCNT_IA_ALL_COUNTERS) + return ia_cntr_names[counter]; + break; + case DAT_HANDLE_TYPE_EP: + if (counter < DCNT_EP_ALL_COUNTERS) + return ep_cntr_names[counter]; + break; + case DAT_HANDLE_TYPE_EVD: + if (counter < DCNT_EVD_ALL_COUNTERS) + return evd_cntr_names[counter]; + break; + default: + return NULL; + } + return NULL; +} + +#include +void dapl_print_counter(DAT_HANDLE dh, int counter, int reset) +{ + int i, max; + DAT_UINT64 *p_cntrs; + DAT_HANDLE_TYPE type = 0; + + dat_get_handle_type(dh, &type); + + switch(type) { + case DAT_HANDLE_TYPE_IA: + max = DCNT_IA_ALL_COUNTERS; + p_cntrs = ((DAPL_IA*)dh)->cntrs; + break; + case DAT_HANDLE_TYPE_EP: + max = DCNT_EP_ALL_COUNTERS; + p_cntrs = ((DAPL_EP*)dh)->cntrs; + break; + case DAT_HANDLE_TYPE_EVD: + max = DCNT_EVD_ALL_COUNTERS; + p_cntrs = ((DAPL_EVD*)dh)->cntrs; + break; + default: + return; + } + + for (i=0; i < max; i++) { + if ((counter == i) || (counter == max)) { + printf( " %s = " F64u " \n", + dapl_query_counter_name(dh, i), + p_cntrs[i] ); + if (reset) p_cntrs[i]=0; + } } - } + return; } #endif /* DAPL_COUNTERS */ diff --git a/dapl/common/dapl_ep_connect.c b/dapl/common/dapl_ep_connect.c index 0c3f10a..9a72531 100755 --- a/dapl/common/dapl_ep_connect.c +++ b/dapl/common/dapl_ep_connect.c @@ -97,7 +97,6 @@ dapl_ep_connect ( private_data, qos, connect_flags); - DAPL_CNTR (DCNT_EP_CONNECT); dat_status = DAT_SUCCESS; ep_ptr = (DAPL_EP *) ep_handle; @@ -124,6 +123,7 @@ dapl_ep_connect ( dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4); goto bail; } + DAPL_CNTR (ep_ptr, DCNT_EP_CONNECT); /* * If the endpoint needs a QP, associated the QP with it. diff --git a/dapl/common/dapl_ep_create.c b/dapl/common/dapl_ep_create.c index ff233b0..8982876 100644 --- a/dapl/common/dapl_ep_create.c +++ b/dapl/common/dapl_ep_create.c @@ -89,7 +89,6 @@ dapl_ep_create ( connect_evd_handle, ep_attr, ep_handle); - DAPL_CNTR(DCNT_EP_CREATE); ia_ptr = (DAPL_IA *)ia_handle; dat_status = DAT_SUCCESS; @@ -102,6 +101,7 @@ dapl_ep_create ( dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA); goto bail; } + DAPL_CNTR(ia_ptr, DCNT_IA_EP_CREATE); /* * Verify non-required parameters. diff --git a/dapl/common/dapl_ep_disconnect.c b/dapl/common/dapl_ep_disconnect.c index fabce92..631d809 100644 --- a/dapl/common/dapl_ep_disconnect.c +++ b/dapl/common/dapl_ep_disconnect.c @@ -75,7 +75,6 @@ dapl_ep_disconnect ( "dapl_ep_disconnect (%p, %x)\n", ep_handle, disconnect_flags); - DAPL_CNTR(DCNT_EP_DISCONNECT); ep_ptr = (DAPL_EP *) ep_handle; @@ -87,6 +86,7 @@ dapl_ep_disconnect ( dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); goto bail; } + DAPL_CNTR(ep_ptr, DCNT_EP_DISCONNECT); /* * Do the verification of parameters and the state change diff --git a/dapl/common/dapl_ep_free.c b/dapl/common/dapl_ep_free.c index 62d9644..76b34f0 100644 --- a/dapl/common/dapl_ep_free.c +++ b/dapl/common/dapl_ep_free.c @@ -73,7 +73,6 @@ dapl_ep_free ( dapl_dbg_log (DAPL_DBG_TYPE_API | DAPL_DBG_TYPE_CM, "dapl_ep_free (%p)\n", ep_handle); - DAPL_CNTR(DCNT_EP_FREE); ep_ptr = (DAPL_EP *) ep_handle; param = &ep_ptr->param; @@ -86,6 +85,7 @@ dapl_ep_free ( dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP); goto bail; } + DAPL_CNTR(ep_ptr->header.owner_ia, DCNT_IA_EP_FREE); if ( ep_ptr->param.ep_state == DAT_EP_STATE_RESERVED || ep_ptr->param.ep_state == DAT_EP_STATE_PASSIVE_CONNECTION_PENDING || diff --git a/dapl/common/dapl_ep_get_status.c b/dapl/common/dapl_ep_get_status.c index 3af7f9a..3f856d3 100644 --- a/dapl/common/dapl_ep_get_status.c +++ b/dapl/common/dapl_ep_get_status.c @@ -99,12 +99,12 @@ dapl_ep_get_status ( if ( in_dto_idle != NULL ) { - *in_dto_idle = (dapls_cb_pending(&ep_ptr->recv_buffer)) ? DAT_FALSE : DAT_TRUE; + *in_dto_idle = dapls_cb_pending(&ep_ptr->recv_buffer); } if ( out_dto_idle != NULL ) { - *out_dto_idle = (dapls_cb_pending(&ep_ptr->req_buffer)) ? DAT_FALSE : DAT_TRUE; + *out_dto_idle = dapls_cb_pending(&ep_ptr->req_buffer); } bail: diff --git a/dapl/common/dapl_ep_post_rdma_read.c b/dapl/common/dapl_ep_post_rdma_read.c index 95a7e05..dc6c39c 100644 --- a/dapl/common/dapl_ep_post_rdma_read.c +++ b/dapl/common/dapl_ep_post_rdma_read.c @@ -86,7 +86,6 @@ dapl_ep_post_rdma_read ( user_cookie.as_64, remote_iov, completion_flags); - DAPL_CNTR(DCNT_POST_RDMA_READ); dat_status = dapl_ep_post_send_req(ep_handle, num_segments, diff --git a/dapl/common/dapl_ep_post_rdma_write.c b/dapl/common/dapl_ep_post_rdma_write.c index 6b9ae94..ef79d0c 100644 --- a/dapl/common/dapl_ep_post_rdma_write.c +++ b/dapl/common/dapl_ep_post_rdma_write.c @@ -85,7 +85,6 @@ dapl_ep_post_rdma_write ( user_cookie.as_64, remote_iov, completion_flags); - DAPL_CNTR(DCNT_POST_RDMA_WRITE); dat_status = dapl_ep_post_send_req(ep_handle, num_segments, diff --git a/dapl/common/dapl_ep_post_recv.c b/dapl/common/dapl_ep_post_recv.c index 12724cd..b11eb96 100644 --- a/dapl/common/dapl_ep_post_recv.c +++ b/dapl/common/dapl_ep_post_recv.c @@ -85,7 +85,6 @@ dapl_ep_post_recv ( local_iov, user_cookie.as_64, completion_flags); - DAPL_CNTR (DCNT_POST_RECV); if ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP) ) { diff --git a/dapl/common/dapl_ep_post_send.c b/dapl/common/dapl_ep_post_send.c index c13a095..0229132 100644 --- a/dapl/common/dapl_ep_post_send.c +++ b/dapl/common/dapl_ep_post_send.c @@ -84,7 +84,6 @@ dapl_ep_post_send ( local_iov, user_cookie.as_64, completion_flags); - DAPL_CNTR(DCNT_POST_SEND); dat_status = dapl_ep_post_send_req(ep_handle, num_segments, diff --git a/dapl/common/dapl_ep_util.c b/dapl/common/dapl_ep_util.c index 782e09a..f75c059 100644 --- a/dapl/common/dapl_ep_util.c +++ b/dapl/common/dapl_ep_util.c @@ -125,6 +125,16 @@ dapl_ep_alloc ( /* zero the structure */ dapl_os_memzero (ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR)); +#ifdef DAPL_COUNTERS + /* Allocate counters */ + ep_ptr->cntrs = dapl_os_alloc(sizeof(DAT_UINT64) * DCNT_EP_ALL_COUNTERS); + if (ep_ptr->cntrs == NULL) { + dapl_os_free(ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR)); + return (NULL); + } + dapl_os_memzero (ep_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_EP_ALL_COUNTERS); +#endif /* DAPL_COUNTERS */ + /* * initialize the header */ @@ -227,6 +237,11 @@ dapl_ep_dealloc ( ep_ptr->ibal_cm_handle = NULL; } #endif + +#ifdef DAPL_COUNTERS + dapl_os_free(ep_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_EP_ALL_COUNTERS); +#endif /* DAPL_COUNTERS */ + dapl_os_free (ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR) ); } diff --git a/dapl/common/dapl_evd_connection_callb.c b/dapl/common/dapl_evd_connection_callb.c index 7f994b0..d5a9750 100644 --- a/dapl/common/dapl_evd_connection_callb.c +++ b/dapl/common/dapl_evd_connection_callb.c @@ -84,8 +84,6 @@ dapl_evd_connection_callback ( ib_cm_event, (void *) ib_cm_handle ); - DAPL_CNTR(DCNT_EVD_CONN_CALLBACK); - /* * Determine the type of handle passed back to us in the context * and sort out key parameters. @@ -101,6 +99,7 @@ dapl_evd_connection_callback ( */ ep_ptr = (DAPL_EP *) context; evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle; + DAPL_CNTR(evd_ptr, DCNT_EVD_CONN_CALLBACK); prd_ptr = (DAPL_PRIVATE *)private_data_ptr; private_data_size = 0; diff --git a/dapl/common/dapl_evd_cq_async_error_callb.c b/dapl/common/dapl_evd_cq_async_error_callb.c index 2b0251a..ca4bc8d 100644 --- a/dapl/common/dapl_evd_cq_async_error_callb.c +++ b/dapl/common/dapl_evd_cq_async_error_callb.c @@ -79,6 +79,7 @@ dapl_evd_cq_async_error_callback ( evd = (DAPL_EVD *) context; async_evd = evd->header.owner_ia->async_error_evd; + DAPL_CNTR(evd->header.owner_ia, DCNT_IA_ASYNC_CQ_ERROR); dat_status = dapls_evd_post_async_error_event( async_evd, diff --git a/dapl/common/dapl_evd_dequeue.c b/dapl/common/dapl_evd_dequeue.c index f6faeeb..5a409f3 100644 --- a/dapl/common/dapl_evd_dequeue.c +++ b/dapl/common/dapl_evd_dequeue.c @@ -75,7 +75,6 @@ DAT_RETURN DAT_API dapl_evd_dequeue ( "dapl_evd_dequeue (%p, %p)\n", evd_handle, event); - DAPL_CNTR(DCNT_EVD_DEQUEUE); evd_ptr = (DAPL_EVD *)evd_handle; dat_status = DAT_SUCCESS; @@ -91,6 +90,7 @@ DAT_RETURN DAT_API dapl_evd_dequeue ( dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); goto bail; } + DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE); /* * We need to dequeue under lock, as the IB OS Access API @@ -122,17 +122,18 @@ DAT_RETURN DAT_API dapl_evd_dequeue ( *event = *local_event; dat_status = dapls_rbuf_add (&evd_ptr->free_event_queue, local_event); - DAPL_CNTR(DCNT_EVD_DEQUEUE_FOUND); + DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE_FOUND); + } else if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) { dat_status = dapls_evd_cq_poll_to_event(evd_ptr, event); - DAPL_CNTR(DCNT_EVD_DEQUEUE_POLL); + DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE_POLL); } else { dat_status = DAT_ERROR (DAT_QUEUE_EMPTY,0); - DAPL_CNTR(DCNT_EVD_DEQUEUE_NOT_FOUND); + DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE_NOT_FOUND); } dapl_os_unlock ( &evd_ptr->header.lock ); diff --git a/dapl/common/dapl_evd_dto_callb.c b/dapl/common/dapl_evd_dto_callb.c index bb3e1f8..347c2e9 100755 --- a/dapl/common/dapl_evd_dto_callb.c +++ b/dapl/common/dapl_evd_dto_callb.c @@ -84,9 +84,9 @@ dapl_evd_dto_callback ( hca_handle, cq_handle, user_context); - DAPL_CNTR(DCNT_EVD_DTO_CALLBACK); evd_ptr = (DAPL_EVD *) user_context; + DAPL_CNTR(evd_ptr, DCNT_EVD_DTO_CALLBACK); dapl_os_assert (hca_handle == evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle); dapl_os_assert (evd_ptr->ib_cq_handle == cq_handle); diff --git a/dapl/common/dapl_evd_free.c b/dapl/common/dapl_evd_free.c index 4d126b2..8e4d082 100755 --- a/dapl/common/dapl_evd_free.c +++ b/dapl/common/dapl_evd_free.c @@ -68,7 +68,6 @@ DAT_RETURN DAT_API dapl_evd_free ( DAT_RETURN dat_status; dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_evd_free (%p)\n", evd_handle); - DAPL_CNTR (DCNT_EVD_FREE); dat_status = DAT_SUCCESS; evd_ptr = (DAPL_EVD *)evd_handle; @@ -79,6 +78,8 @@ DAT_RETURN DAT_API dapl_evd_free ( goto bail; } + DAPL_CNTR(evd_ptr->header.owner_ia, DCNT_IA_EVD_FREE); + if (dapl_os_atomic_read (&evd_ptr->evd_ref_count) != 0) { dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_EVD_IN_USE); diff --git a/dapl/common/dapl_evd_qp_async_error_callb.c b/dapl/common/dapl_evd_qp_async_error_callb.c index b14db3c..27b6ddc 100644 --- a/dapl/common/dapl_evd_qp_async_error_callb.c +++ b/dapl/common/dapl_evd_qp_async_error_callb.c @@ -114,6 +114,7 @@ dapl_evd_qp_async_error_callback ( ia_ptr = ep_ptr->header.owner_ia; async_evd = (DAPL_EVD *) ia_ptr->async_error_evd; + DAPL_CNTR(ia_ptr, DCNT_IA_ASYNC_QP_ERROR); dapl_dbg_log ( DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION, diff --git a/dapl/common/dapl_evd_un_async_error_callb.c b/dapl/common/dapl_evd_un_async_error_callb.c index 5ba0f3e..987aee2 100644 --- a/dapl/common/dapl_evd_un_async_error_callb.c +++ b/dapl/common/dapl_evd_un_async_error_callb.c @@ -78,6 +78,7 @@ dapl_evd_un_async_error_callback ( } async_evd = (DAPL_EVD *) context; + DAPL_CNTR(async_evd->header.owner_ia, DCNT_IA_ASYNC_ERROR); dat_status = dapls_ib_get_async_event(cause_ptr, &async_event); diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c index c6c7463..ad3d13d 100644 --- a/dapl/common/dapl_evd_util.c +++ b/dapl/common/dapl_evd_util.c @@ -273,6 +273,17 @@ dapls_evd_alloc ( /* zero the structure */ dapl_os_memzero (evd_ptr, sizeof (DAPL_EVD)); +#ifdef DAPL_COUNTERS + /* Allocate counters */ + evd_ptr->cntrs = dapl_os_alloc(sizeof(DAT_UINT64) * DCNT_EVD_ALL_COUNTERS); + if (evd_ptr->cntrs == NULL) { + dapl_os_free(evd_ptr, sizeof(DAPL_EVD)); + return (NULL); + } + dapl_os_memzero(evd_ptr->cntrs, + sizeof(DAT_UINT64) * DCNT_EVD_ALL_COUNTERS); +#endif /* DAPL_COUNTERS */ + /* * initialize the header */ @@ -542,6 +553,10 @@ dapls_evd_dealloc ( } #endif +#ifdef DAPL_COUNTERS + dapl_os_free(evd_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_EVD_ALL_COUNTERS); +#endif /* DAPL_COUNTERS */ + dapl_os_free (evd_ptr, sizeof (DAPL_EVD)); bail: diff --git a/dapl/common/dapl_ia_util.c b/dapl/common/dapl_ia_util.c index 7b03afc..a42063b 100755 --- a/dapl/common/dapl_ia_util.c +++ b/dapl/common/dapl_ia_util.c @@ -78,6 +78,16 @@ dapl_ia_alloc ( DAT_PROVIDER * provider, DAPL_HCA * hca_ptr ) /* zero the structure */ dapl_os_memzero (ia_ptr, sizeof (DAPL_IA)); +#ifdef DAPL_COUNTERS + /* Allocate counters */ + ia_ptr->cntrs = dapl_os_alloc(sizeof(DAT_UINT64) * DCNT_IA_ALL_COUNTERS); + if (ia_ptr->cntrs == NULL) { + dapl_os_free (ia_ptr, sizeof(DAPL_IA)); + return (NULL); + } + dapl_os_memzero(ia_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_IA_ALL_COUNTERS); +#endif /* DAPL_COUNTERS */ + /* * initialize the header */ @@ -596,6 +606,10 @@ dapls_ia_free ( DAPL_IA *ia_ptr ) ia_ptr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */ dapl_os_lock_destroy (&ia_ptr->header.lock); +#ifdef DAPL_COUNTERS + dapl_os_free(ia_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_IA_ALL_COUNTERS); +#endif /* DAPL_COUNTERS */ + dapl_os_free (ia_ptr, sizeof (DAPL_IA)); } diff --git a/dapl/common/dapl_lmr_free.c b/dapl/common/dapl_lmr_free.c index f4106b4..3fa67b2 100644 --- a/dapl/common/dapl_lmr_free.c +++ b/dapl/common/dapl_lmr_free.c @@ -62,6 +62,7 @@ dapl_lmr_free ( IN DAT_LMR_HANDLE lmr_handle ) { DAPL_LMR *lmr; + DAPL_PZ *pz; DAT_RETURN dat_status; dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_lmr_free (%p)\n", lmr_handle); @@ -72,7 +73,10 @@ dapl_lmr_free ( goto bail; } - lmr = (DAPL_LMR *) lmr_handle; + lmr = (DAPL_LMR *)lmr_handle; + pz = (DAPL_PZ *)lmr->param.pz_handle; + + DAPL_CNTR(pz->header.owner_ia, DCNT_IA_LMR_FREE); switch (lmr->param.mem_type) { @@ -85,8 +89,6 @@ dapl_lmr_free ( case DAT_MEM_TYPE_VIRTUAL: case DAT_MEM_TYPE_LMR: { - DAPL_PZ *pz; - if ( 0 != dapl_os_atomic_read (&lmr->lmr_ref_count) ) { return DAT_INVALID_STATE; @@ -103,10 +105,7 @@ dapl_lmr_free ( if (dat_status == DAT_SUCCESS) { - - pz = (DAPL_PZ *) lmr->param.pz_handle; dapl_os_atomic_dec (&pz->pz_ref_count); - dapl_lmr_dealloc (lmr); } else diff --git a/dapl/common/dapl_psp_create.c b/dapl/common/dapl_psp_create.c index 9d2e945..a6eb31c 100644 --- a/dapl/common/dapl_psp_create.c +++ b/dapl/common/dapl_psp_create.c @@ -114,7 +114,8 @@ dapl_psp_create ( dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG4); goto bail; } - + + DAPL_CNTR(ia_ptr, DCNT_IA_PSP_CREATE); /* * See if we have a quiescent listener to use for this PSP, else diff --git a/dapl/common/dapl_psp_create_any.c b/dapl/common/dapl_psp_create_any.c index e2faa4a..9b9b575 100644 --- a/dapl/common/dapl_psp_create_any.c +++ b/dapl/common/dapl_psp_create_any.c @@ -133,6 +133,8 @@ dapl_psp_create_any ( goto bail; } + DAPL_CNTR(ia_ptr, DCNT_IA_PSP_CREATE_ANY); + /* * Fill out the args for a PSP */ diff --git a/dapl/common/dapl_psp_free.c b/dapl/common/dapl_psp_free.c index 28913b4..0117e19 100644 --- a/dapl/common/dapl_psp_free.c +++ b/dapl/common/dapl_psp_free.c @@ -82,6 +82,8 @@ dapl_psp_free ( ia_ptr = sp_ptr->header.owner_ia; + DAPL_CNTR(ia_ptr->header.owner_ia, DCNT_IA_PSP_FREE); + /* * Remove the connection listener if it has been established * and there are no current connections in progress. diff --git a/dapl/common/dapl_pz_create.c b/dapl/common/dapl_pz_create.c index a7105aa..efca311 100644 --- a/dapl/common/dapl_pz_create.c +++ b/dapl/common/dapl_pz_create.c @@ -90,6 +90,7 @@ dapl_pz_create ( dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); goto bail; } + DAPL_CNTR(ia, DCNT_IA_PZ_CREATE); dat_status = dapls_ib_pd_alloc (ia, pz); if ( DAT_SUCCESS != dat_status ) diff --git a/dapl/common/dapl_pz_free.c b/dapl/common/dapl_pz_free.c index b2510f7..270dd38 100644 --- a/dapl/common/dapl_pz_free.c +++ b/dapl/common/dapl_pz_free.c @@ -72,6 +72,8 @@ dapl_pz_free ( pz = (DAPL_PZ *) pz_handle; + DAPL_CNTR(pz->header.owner_ia, DCNT_IA_PZ_FREE); + if ( 0 != dapl_os_atomic_read (&pz->pz_ref_count) ) { dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_PZ_IN_USE); diff --git a/dapl/common/dapl_rmr_create.c b/dapl/common/dapl_rmr_create.c index 9a7dbd9..d3e1661 100755 --- a/dapl/common/dapl_rmr_create.c +++ b/dapl/common/dapl_rmr_create.c @@ -71,7 +71,6 @@ dapl_rmr_create ( } pz = (DAPL_PZ *) pz_handle; - rmr = dapl_rmr_alloc (pz); if ( rmr == NULL ) @@ -79,6 +78,7 @@ dapl_rmr_create ( dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY); goto bail; } + DAPL_CNTR(pz->header.owner_ia, DCNT_IA_RMR_CREATE); dat_status = dapls_ib_mw_alloc (rmr); diff --git a/dapl/common/dapl_rmr_free.c b/dapl/common/dapl_rmr_free.c index a4edb13..f75aaed 100644 --- a/dapl/common/dapl_rmr_free.c +++ b/dapl/common/dapl_rmr_free.c @@ -58,6 +58,7 @@ dapl_rmr_free ( IN DAT_RMR_HANDLE rmr_handle ) { DAPL_RMR *rmr; + DAPL_PZ *pz; DAT_RETURN dat_status; dat_status = DAT_SUCCESS; @@ -69,6 +70,8 @@ dapl_rmr_free ( } rmr = (DAPL_RMR *) rmr_handle; + pz = (DAPL_PZ *)rmr->param.pz_handle; + DAPL_CNTR(pz->header.owner_ia, DCNT_IA_RMR_FREE); /* * If the user did not perform an unbind op, release diff --git a/dapl/common/dapl_rsp_create.c b/dapl/common/dapl_rsp_create.c index a9c9b5f..cf41dd9 100644 --- a/dapl/common/dapl_rsp_create.c +++ b/dapl/common/dapl_rsp_create.c @@ -126,6 +126,8 @@ dapl_rsp_create ( goto bail; } + DAPL_CNTR(ia_ptr, DCNT_IA_RSP_CREATE); + sp_ptr = dapls_ia_sp_search (ia_ptr, conn_qual, DAT_FALSE); sp_found = DAT_TRUE; if (sp_ptr == NULL) diff --git a/dapl/common/dapl_rsp_free.c b/dapl/common/dapl_rsp_free.c index b7bc40c..e016f0d 100644 --- a/dapl/common/dapl_rsp_free.c +++ b/dapl/common/dapl_rsp_free.c @@ -85,6 +85,8 @@ dapl_rsp_free ( /* ia_ptr = (DAPL_IA *)sp_ptr->header.owner_ia; */ ia_ptr = sp_ptr->header.owner_ia; + DAPL_CNTR(ia_ptr, DCNT_IA_RSP_FREE); + /* * Remove the connection listener if there are no connections. If * we defer removing the sp it becomes something of a zombie diff --git a/dapl/common/dapl_srq_create.c b/dapl/common/dapl_srq_create.c index b03bbd6..ef45826 100644 --- a/dapl/common/dapl_srq_create.c +++ b/dapl/common/dapl_srq_create.c @@ -97,6 +97,8 @@ dapl_srq_create ( goto bail; } + DAPL_CNTR(ia_ptr, DCNT_IA_SRQ_CREATE); + /* * Verify non-required parameters. * N.B. Assumption: any parameter that can be diff --git a/dapl/common/dapl_srq_free.c b/dapl/common/dapl_srq_free.c index 55d74fc..9decf54 100644 --- a/dapl/common/dapl_srq_free.c +++ b/dapl/common/dapl_srq_free.c @@ -96,6 +96,8 @@ dapl_srq_free ( ia_ptr = srq_ptr->header.owner_ia; + DAPL_CNTR(ia_ptr, DCNT_IA_SRQ_FREE); + /* * Do verification of parameters and the state change atomically. */ diff --git a/dapl/common/dapl_timer_util.c b/dapl/common/dapl_timer_util.c index c7407c1..12c41bc 100644 --- a/dapl/common/dapl_timer_util.c +++ b/dapl/common/dapl_timer_util.c @@ -107,7 +107,6 @@ dapls_timer_set ( DAPL_OS_TIMEVAL cur_time; DAT_BOOLEAN wakeup_tmo_thread; - DAPL_CNTR(DCNT_TIMER_SET); /* * Start the timer thread the first time we need a timer */ @@ -216,7 +215,6 @@ void dapls_timer_cancel ( IN DAPL_OS_TIMER *timer) { - DAPL_CNTR(DCNT_TIMER_CANCEL); dapl_os_lock ( &g_daplTimerHead.lock ); /* * make sure the entry has not been removed by another thread diff --git a/dapl/ibal/dapl_ibal_extensions.c b/dapl/ibal/dapl_ibal_extensions.c index b05c0bb..643125d 100644 --- a/dapl/ibal/dapl_ibal_extensions.c +++ b/dapl/ibal/dapl_ibal_extensions.c @@ -93,8 +93,6 @@ dapl_extensions(IN DAT_HANDLE dat_handle, "dapl_extensions(hdl %p operation %d, ...)\n", dat_handle, ext_op); - DAPL_CNTR(DCNT_EXTENSION); - switch ((int)ext_op) { diff --git a/dapl/include/dapl.h b/dapl/include/dapl.h index e2025ce..b13c963 100755 --- a/dapl/include/dapl.h +++ b/dapl/include/dapl.h @@ -313,6 +313,9 @@ struct dapl_ia DAPL_LLIST_HEAD psp_list_head; /* PSP queue */ DAPL_LLIST_HEAD rsp_list_head; /* RSP queue */ DAPL_LLIST_HEAD srq_list_head; /* SRQ queue */ +#ifdef DAPL_COUNTERS + void *cntrs; +#endif }; /* DAPL_CNO maps to DAT_CNO_HANDLE */ @@ -388,6 +391,10 @@ struct dapl_evd DAT_COUNT threshold; DAPL_EVD_COMPLETION completion_type; + +#ifdef DAPL_COUNTERS + void *cntrs; +#endif }; /* DAPL_PRIVATE used to pass private data in a connection */ @@ -472,6 +479,9 @@ struct dapl_ep DAT_BOOLEAN sent_discreq; dp_ib_cm_handle_t ibal_cm_handle; #endif +#ifdef DAPL_COUNTERS + void *cntrs; +#endif }; /* DAPL_SRQ maps to DAT_SRQ_HANDLE */ diff --git a/dapl/include/dapl_debug.h b/dapl/include/dapl_debug.h index f0de7c8..d034227 100644 --- a/dapl/include/dapl_debug.h +++ b/dapl/include/dapl_debug.h @@ -99,50 +99,25 @@ static __inline void dapl_dbg_log ( DAPL_DBG_TYPE type, const char *fmt, ...) #endif /* !DAPL_DBG */ -/* - * Counters - */ -#define DCNT_EP_CREATE 0 -#define DCNT_EP_FREE 1 -#define DCNT_EP_CONNECT 2 -#define DCNT_EP_DISCONNECT 3 -#define DCNT_POST_SEND 4 -#define DCNT_POST_RECV 5 -#define DCNT_POST_RDMA_WRITE 6 -#define DCNT_POST_RDMA_READ 7 -#define DCNT_EVD_CREATE 8 -#define DCNT_EVD_FREE 9 -#define DCNT_EVD_WAIT 10 -#define DCNT_EVD_WAIT_BLOCKED 11 -#define DCNT_EVD_WAIT_CMP_NTFY 12 -#define DCNT_EVD_DTO_CALLBACK 13 -#define DCNT_EVD_CONN_CALLBACK 14 -#define DCNT_EVD_DEQUEUE 15 -#define DCNT_EVD_DEQUEUE_POLL 16 -#define DCNT_EVD_DEQUEUE_FOUND 17 -#define DCNT_EVD_DEQUEUE_NOT_FOUND 18 -#define DCNT_TIMER_SET 19 -#define DCNT_TIMER_CANCEL 20 -#define DCNT_EXTENSION 21 -#define DCNT_NUM_COUNTERS 22 -#define DCNT_ALL_COUNTERS DCNT_NUM_COUNTERS - -#if defined(DAPL_COUNTERS) - -extern void dapl_dump_cntr( int cntr ); -extern int dapl_dbg_counters[]; - -#define DAPL_CNTR(cntr) dapl_os_atomic_inc (&dapl_dbg_counters[cntr]); -#define DAPL_DUMP_CNTR(cntr) dapl_dump_cntr( cntr ); -#define DAPL_COUNTERS_INIT() +#include + +#ifdef DAPL_COUNTERS + +#define DAPL_CNTR(h_ptr, cntr) ((DAT_UINT64*)h_ptr->cntrs)[cntr]++ +#define DAPL_CNTR_DATA(h_ptr, cntr, data) ((DAT_UINT64*)h_ptr->cntrs)[cntr]+= data + +DAT_RETURN dapl_query_counter(DAT_HANDLE dh, + int counter, + void *p_cntrs_out, + int reset); +char *dapl_query_counter_name(DAT_HANDLE dh, int counter); +void dapl_print_counter(DAT_HANDLE dh, int counter, int reset); #else -#define DAPL_CNTR(cntr) -#define DAPL_DUMP_CNTR(cntr) -#define DAPL_COUNTERS_INIT() +#define DAPL_CNTR(handle, cntr) +#define DAPL_CNTR_DATA(handle, cntr, data) #endif /* DAPL_COUNTERS */ - #endif /* _DAPL_DEBUG_H_ */ diff --git a/dapl/openib_cma/dapl_ib_dto.h b/dapl/openib_cma/dapl_ib_dto.h index 2b01963..eba19b7 100644 --- a/dapl/openib_cma/dapl_ib_dto.h +++ b/dapl/openib_cma/dapl_ib_dto.h @@ -89,7 +89,7 @@ dapls_ib_post_recv ( if (NULL == ds_array_p) return (DAT_INSUFFICIENT_RESOURCES); - + /* setup work request */ total_len = 0; wr.next = 0; @@ -126,6 +126,9 @@ dapls_ib_post_recv ( if (ret) return( dapl_convert_errno(errno,"ibv_recv") ); + DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len); + return DAT_SUCCESS; } @@ -237,6 +240,25 @@ dapls_ib_post_send ( if (ret) return( dapl_convert_errno(errno,"ibv_send") ); +#ifdef DAPL_COUNTERS + switch (op_type) { + case OP_SEND: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len); + break; + case OP_RDMA_WRITE: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len); + break; + case OP_RDMA_READ: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len); + break; + default: + break; + } +#endif /* DAPL_COUNTERS */ + dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); return DAT_SUCCESS; } @@ -416,7 +438,25 @@ dapls_ib_post_ext_send ( if (ret) return( dapl_convert_errno(errno,"ibv_send") ); - + +#ifdef DAPL_COUNTERS + switch (op_type) { + case OP_RDMA_WRITE_IMM: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM); + DAPL_CNTR_DATA(ep_ptr, + DCNT_EP_POST_WRITE_IMM_DATA, total_len); + break; + case OP_COMP_AND_SWAP: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP); + break; + case OP_FETCH_AND_ADD: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD); + break; + default: + break; + } +#endif /* DAPL_COUNTERS */ + dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); return DAT_SUCCESS; } @@ -434,6 +474,10 @@ dapls_ib_optional_prv_dat( /* map Work Completions to DAPL WR operations */ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p) { +#ifdef DAPL_COUNTERS + DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id; +#endif /* DAPL_COUNTERS */ + switch (cqe_p->opcode) { case IBV_WC_SEND: return (OP_SEND); @@ -451,11 +495,21 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p) case IBV_WC_BIND_MW: return (OP_BIND_MW); case IBV_WC_RECV: - if (cqe_p->wc_flags & IBV_WC_WITH_IMM) + if (cqe_p->wc_flags & IBV_WC_WITH_IMM) { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, + cqe_p->byte_len); return (OP_RECEIVE_IMM); - else + } else { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, + cqe_p->byte_len); return (OP_RECEIVE); + } case IBV_WC_RECV_RDMA_WITH_IMM: + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, + cqe_p->byte_len); return (OP_RECEIVE_IMM); default: return (OP_INVALID); diff --git a/dapl/openib_cma/dapl_ib_extensions.c b/dapl/openib_cma/dapl_ib_extensions.c index 1402057..af34128 100755 --- a/dapl/openib_cma/dapl_ib_extensions.c +++ b/dapl/openib_cma/dapl_ib_extensions.c @@ -94,8 +94,6 @@ dapl_extensions(IN DAT_HANDLE dat_handle, "dapl_extensions(hdl %p operation %d, ...)\n", dat_handle, ext_op); - DAPL_CNTR(DCNT_EXTENSION); - switch ((int)ext_op) { @@ -150,6 +148,39 @@ dapl_extensions(IN DAT_HANDLE dat_handle, break; +#ifdef DAPL_COUNTERS + case DAT_QUERY_COUNTERS_OP: + { + int cntr, reset; + DAT_UINT64 * p_cntr_out; + + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " Query counter extension call\n"); + + cntr = va_arg(args, int); + p_cntr_out = va_arg(args, DAT_UINT64*); + reset = va_arg(args, int); + + status = dapl_query_counter(dat_handle, cntr, + p_cntr_out, reset); + break; + } + case DAT_PRINT_COUNTERS_OP: + { + int cntr, reset; + + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " Print counter extension call\n"); + + cntr = va_arg(args, int); + reset = va_arg(args, int); + + dapl_print_counter(dat_handle, cntr, reset); + status = DAT_SUCCESS; + break; + } +#endif /* DAPL_COUNTERS */ + default: dapl_dbg_log(DAPL_DBG_TYPE_ERR, "unsupported extension(%d)\n", (int)ext_op); diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c index 36b534e..c5641f0 100755 --- a/dapl/openib_cma/dapl_ib_util.c +++ b/dapl/openib_cma/dapl_ib_util.c @@ -970,6 +970,11 @@ DAT_NAMED_ATTR ib_attrs[] = { { DAT_IB_ATTR_IMMED_DATA, "TRUE" }, +#ifdef DAPL_COUNTERS + { + DAT_ATTR_COUNTERS, "TRUE" + }, +#endif /* DAPL_COUNTERS */ #endif }; diff --git a/dapl/openib_scm/dapl_ib_dto.h b/dapl/openib_scm/dapl_ib_dto.h index fa19d01..ff338fc 100644 --- a/dapl/openib_scm/dapl_ib_dto.h +++ b/dapl/openib_scm/dapl_ib_dto.h @@ -92,7 +92,7 @@ dapls_ib_post_recv ( if (NULL == ds_array_p) return (DAT_INSUFFICIENT_RESOURCES); - + /* setup work request */ total_len = 0; wr.next = 0; @@ -129,6 +129,9 @@ dapls_ib_post_recv ( if (ret) return( dapl_convert_errno(errno,"ibv_recv") ); + DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len); + return DAT_SUCCESS; } @@ -244,6 +247,25 @@ dapls_ib_post_send ( if (ret) return( dapl_convert_errno(errno,"ibv_send") ); +#ifdef DAPL_COUNTERS + switch (op_type) { + case OP_SEND: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len); + break; + case OP_RDMA_WRITE: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len); + break; + case OP_RDMA_READ: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len); + break; + default: + break; + } +#endif /* DAPL_COUNTERS */ + dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); return DAT_SUCCESS; } @@ -452,8 +474,30 @@ dapls_ib_post_ext_send ( if (ret) return( dapl_convert_errno(errno,"ibv_send") ); +#ifdef DAPL_COUNTERS + switch (op_type) { + case OP_RDMA_WRITE_IMM: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM); + DAPL_CNTR_DATA(ep_ptr, + DCNT_EP_POST_WRITE_IMM_DATA, total_len); + break; + case OP_COMP_AND_SWAP: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP); + break; + case OP_FETCH_AND_ADD: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD); + break; + case OP_SEND_UD: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND_UD); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_UD_DATA, total_len); + break; + default: + break; + } +#endif /* DAPL_COUNTERS */ + dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); - return DAT_SUCCESS; + return DAT_SUCCESS; } #endif @@ -470,6 +514,10 @@ dapls_ib_optional_prv_dat( /* map Work Completions to DAPL WR operations */ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p) { +#ifdef DAPL_COUNTERS + DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id; +#endif /* DAPL_COUNTERS */ + switch (cqe_p->opcode) { case IBV_WC_SEND: if (CQE_WR_TYPE_UD(cqe_p->wr_id)) @@ -490,13 +538,27 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p) case IBV_WC_BIND_MW: return (OP_BIND_MW); case IBV_WC_RECV: - if (CQE_WR_TYPE_UD(cqe_p->wr_id)) + if (CQE_WR_TYPE_UD(cqe_p->wr_id)) { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_UD); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_UD_DATA, + cqe_p->byte_len); return (OP_RECV_UD); - else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) - return (OP_RECEIVE_MSG_IMM); - else + } + else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, + cqe_p->byte_len); + return (OP_RECEIVE_IMM); + } else { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, + cqe_p->byte_len); return (OP_RECEIVE); + } case IBV_WC_RECV_RDMA_WITH_IMM: + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, + cqe_p->byte_len); return (OP_RECEIVE_IMM); default: return (OP_INVALID); diff --git a/dapl/openib_scm/dapl_ib_extensions.c b/dapl/openib_scm/dapl_ib_extensions.c index 692c8a8..1d23ca3 100755 --- a/dapl/openib_scm/dapl_ib_extensions.c +++ b/dapl/openib_scm/dapl_ib_extensions.c @@ -96,8 +96,6 @@ dapl_extensions(IN DAT_HANDLE dat_handle, "dapl_extensions(hdl %p operation %d, ...)\n", dat_handle, ext_op); - DAPL_CNTR(DCNT_EXTENSION); - switch ((int)ext_op) { @@ -167,6 +165,39 @@ dapl_extensions(IN DAT_HANDLE dat_handle, OP_SEND_UD, comp_flags, ah); break; +#ifdef DAPL_COUNTERS + case DAT_QUERY_COUNTERS_OP: + { + int cntr, reset; + DAT_UINT64 * p_cntr_out; + + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " Query counter extension call\n"); + + cntr = va_arg(args, int); + p_cntr_out = va_arg(args, DAT_UINT64*); + reset = va_arg(args, int); + + status = dapl_query_counter(dat_handle, cntr, + p_cntr_out, reset); + break; + } + case DAT_PRINT_COUNTERS_OP: + { + int cntr, reset; + + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " Print counter extension call\n"); + + cntr = va_arg(args, int); + reset = va_arg(args, int); + + dapl_print_counter(dat_handle, cntr, reset); + status = DAT_SUCCESS; + break; + } +#endif /* DAPL_COUNTERS */ + default: dapl_dbg_log(DAPL_DBG_TYPE_ERR, "unsupported extension(%d)\n", (int)ext_op); diff --git a/dapl/openib_scm/dapl_ib_util.c b/dapl/openib_scm/dapl_ib_util.c index d82d3f5..7623529 100644 --- a/dapl/openib_scm/dapl_ib_util.c +++ b/dapl/openib_scm/dapl_ib_util.c @@ -618,6 +618,11 @@ DAT_NAMED_ATTR ib_attrs[] = { { DAT_IB_ATTR_UD, "TRUE" }, +#ifdef DAPL_COUNTERS + { + DAT_ATTR_COUNTERS, "TRUE" + }, +#endif /* DAPL_COUNTERS */ #endif }; diff --git a/dapl/udapl/dapl_evd_create.c b/dapl/udapl/dapl_evd_create.c index 7687a81..5ca5c22 100644 --- a/dapl/udapl/dapl_evd_create.c +++ b/dapl/udapl/dapl_evd_create.c @@ -95,7 +95,6 @@ DAT_RETURN DAT_API dapl_evd_create ( cno_handle, evd_flags, evd_handle); - DAPL_CNTR(DCNT_EVD_CREATE); ia_ptr = (DAPL_IA *)ia_handle; cno_ptr = (DAPL_CNO *)cno_handle; @@ -109,6 +108,8 @@ DAT_RETURN DAT_API dapl_evd_create ( goto bail; } + DAPL_CNTR(ia_ptr, DCNT_IA_EVD_CREATE); + if ( evd_min_qlen <= 0 ) { dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2); diff --git a/dapl/udapl/dapl_evd_wait.c b/dapl/udapl/dapl_evd_wait.c index 578041a..9fc0ba2 100644 --- a/dapl/udapl/dapl_evd_wait.c +++ b/dapl/udapl/dapl_evd_wait.c @@ -85,7 +85,6 @@ DAT_RETURN DAT_API dapl_evd_wait ( threshold, event, nmore); - DAPL_CNTR(DCNT_EVD_WAIT); evd_ptr = (DAPL_EVD *)evd_handle; dat_status = DAT_SUCCESS; @@ -122,6 +121,7 @@ DAT_RETURN DAT_API dapl_evd_wait ( dat_status = DAT_ERROR (DAT_INVALID_STATE,0); goto bail; } + DAPL_CNTR(evd_ptr, DCNT_EVD_WAIT); dapl_dbg_log (DAPL_DBG_TYPE_EVD, "dapl_evd_wait: EVD %p, CQ %p\n", @@ -206,7 +206,7 @@ DAT_RETURN DAT_API dapl_evd_wait ( (evd_ptr->completion_type == DAPL_EVD_STATE_SOLICITED_WAIT) ? IB_NOTIFY_ON_SOLIC_COMP : IB_NOTIFY_ON_NEXT_COMP ); - DAPL_CNTR(DCNT_EVD_WAIT_CMP_NTFY); + DAPL_CNTR(evd_ptr, DCNT_EVD_WAIT_NOTIFY); /* FIXME report error */ dapl_os_assert(dat_status == DAT_SUCCESS); @@ -225,7 +225,7 @@ DAT_RETURN DAT_API dapl_evd_wait ( */ evd_ptr->threshold = threshold; - DAPL_CNTR(DCNT_EVD_WAIT_BLOCKED); + DAPL_CNTR(evd_ptr, DCNT_EVD_WAIT_BLOCKED); #ifdef CQ_WAIT_OBJECT if (evd_ptr->cq_wait_obj_handle) diff --git a/dapl/udapl/dapl_init.c b/dapl/udapl/dapl_init.c index a4afba5..f19468b 100644 --- a/dapl/udapl/dapl_init.c +++ b/dapl/udapl/dapl_init.c @@ -115,8 +115,6 @@ void dapl_init ( void ) goto bail; } - DAPL_COUNTERS_INIT(); - return; bail: diff --git a/dapl/udapl/dapl_lmr_create.c b/dapl/udapl/dapl_lmr_create.c index 99b184a..dd93445 100644 --- a/dapl/udapl/dapl_lmr_create.c +++ b/dapl/udapl/dapl_lmr_create.c @@ -489,6 +489,8 @@ dapl_lmr_create ( ia = (DAPL_IA *) ia_handle; pz = (DAPL_PZ *) pz_handle; + DAPL_CNTR(ia, DCNT_IA_LMR_CREATE); + switch (mem_type) { case DAT_MEM_TYPE_VIRTUAL: diff --git a/dapl/udapl/linux/dapl_osd.h b/dapl/udapl/linux/dapl_osd.h index ae02944..974a0f0 100644 --- a/dapl/udapl/linux/dapl_osd.h +++ b/dapl/udapl/linux/dapl_osd.h @@ -539,10 +539,11 @@ DAT_RETURN dapl_os_get_time (DAPL_OS_TIMEVAL *); * ability to define 64 bit formats, which unfortunatly are non standard * in the C compiler world. E.g. %llx for gcc, %I64x for Windows */ -#define F64d "%lld" -#define F64u "%llu" -#define F64x "%llx" -#define F64X "%llX" +#include +#define F64d "%"PRId64 +#define F64u "%"PRIu64 +#define F64x "%"PRIx64 +#define F64X "%"PRIX64 /* diff --git a/dat/include/dat2/dat_ib_extensions.h b/dat/include/dat2/dat_ib_extensions.h index eb10714..2c082b1 100755 --- a/dat/include/dat2/dat_ib_extensions.h +++ b/dat/include/dat2/dat_ib_extensions.h @@ -58,7 +58,21 @@ * DAT_NAMED_ATTR name == extended operations and version, * version_value = version number of extension API */ -#define DAT_IB_EXTENSION_VERSION 202 /* 2.0.2 */ + +/* 2.0.1 - Initial IB extension support, atomic and immed data + * dat_ib_post_fetch_and_add() + * dat_ib_post_cmp_and_swap() + * dat_ib_post_rdma_write_immed() + * + * 2.0.2 - Add UD support, post send and remote_ah via connect events + * dat_ib_post_send_ud() + * + * 2.0.3 - Add query/print counter support for IA, EP, and EVD's + * dat_query_counters(), dat_print_counters() + * + */ +#define DAT_IB_EXTENSION_VERSION 203 /* 2.0.3 */ +#define DAT_ATTR_COUNTERS "DAT_COUNTERS" #define DAT_IB_ATTR_FETCH_AND_ADD "DAT_IB_FETCH_AND_ADD" #define DAT_IB_ATTR_CMP_AND_SWAP "DAT_IB_CMP_AND_SWAP" #define DAT_IB_ATTR_IMMED_DATA "DAT_IB_IMMED_DATA" @@ -87,7 +101,9 @@ typedef enum dat_ib_op DAT_IB_FETCH_AND_ADD_OP, DAT_IB_CMP_AND_SWAP_OP, DAT_IB_RDMA_WRITE_IMMED_OP, - DAT_IB_UD_SEND_OP + DAT_IB_UD_SEND_OP, + DAT_QUERY_COUNTERS_OP, + DAT_PRINT_COUNTERS_OP } DAT_IB_OP; @@ -247,6 +263,92 @@ typedef enum dat_ib_service_type } DAT_IB_SERVICE_TYPE; +/* + * Definitions for 64-bit IA Counters + */ +typedef enum dat_ia_counters +{ + DCNT_IA_PZ_CREATE, + DCNT_IA_PZ_FREE, + DCNT_IA_LMR_CREATE, + DCNT_IA_LMR_FREE, + DCNT_IA_RMR_CREATE, + DCNT_IA_RMR_FREE, + DCNT_IA_PSP_CREATE, + DCNT_IA_PSP_CREATE_ANY, + DCNT_IA_PSP_FREE, + DCNT_IA_RSP_CREATE, + DCNT_IA_RSP_FREE, + DCNT_IA_EVD_CREATE, + DCNT_IA_EVD_FREE, + DCNT_IA_EP_CREATE, + DCNT_IA_EP_FREE, + DCNT_IA_SRQ_CREATE, + DCNT_IA_SRQ_FREE, + DCNT_IA_SP_CR, + DCNT_IA_SP_CR_ACCEPTED, + DCNT_IA_SP_CR_REJECTED, + DCNT_IA_MEM_ALLOC, + DCNT_IA_MEM_ALLOC_DATA, + DCNT_IA_MEM_FREE, + DCNT_IA_ASYNC_ERROR, + DCNT_IA_ASYNC_QP_ERROR, + DCNT_IA_ASYNC_CQ_ERROR, + DCNT_IA_ALL_COUNTERS, /* MUST be last */ + +} DAT_IA_COUNTERS; + +/* + * Definitions for 64-bit EP Counters + */ +typedef enum dat_ep_counters +{ + DCNT_EP_CONNECT, + DCNT_EP_DISCONNECT, + DCNT_EP_POST_SEND, + DCNT_EP_POST_SEND_DATA, + DCNT_EP_POST_SEND_UD, + DCNT_EP_POST_SEND_UD_DATA, + DCNT_EP_POST_RECV, + DCNT_EP_POST_RECV_DATA, + DCNT_EP_POST_WRITE, + DCNT_EP_POST_WRITE_DATA, + DCNT_EP_POST_WRITE_IMM, + DCNT_EP_POST_WRITE_IMM_DATA, + DCNT_EP_POST_READ, + DCNT_EP_POST_READ_DATA, + DCNT_EP_POST_CMP_SWAP, + DCNT_EP_POST_FETCH_ADD, + DCNT_EP_RECV, + DCNT_EP_RECV_DATA, + DCNT_EP_RECV_UD, + DCNT_EP_RECV_UD_DATA, + DCNT_EP_RECV_IMM, + DCNT_EP_RECV_IMM_DATA, + DCNT_EP_RECV_RDMA_IMM, + DCNT_EP_RECV_RDMA_IMM_DATA, + DCNT_EP_ALL_COUNTERS, /* MUST be last */ + +} DAT_EP_COUNTERS; + +/* + * Definitions for 64-bit EVD Counters + */ +typedef enum dat_evd_counters +{ + DCNT_EVD_WAIT, + DCNT_EVD_WAIT_BLOCKED, + DCNT_EVD_WAIT_NOTIFY, + DCNT_EVD_DEQUEUE, + DCNT_EVD_DEQUEUE_FOUND, + DCNT_EVD_DEQUEUE_NOT_FOUND, + DCNT_EVD_DEQUEUE_POLL, + DCNT_EVD_DEQUEUE_POLL_FOUND, + DCNT_EVD_CONN_CALLBACK, + DCNT_EVD_DTO_CALLBACK, + DCNT_EVD_ALL_COUNTERS, /* MUST be last */ + +} DAT_EVD_COUNTERS; /* Extended RETURN and EVENT STATUS string helper functions */ @@ -370,5 +472,32 @@ dat_strerror_ext_status ( (cookie), \ (flgs)) + +/* + * Query counter(s): + * Provide IA, EP, or EVD and call will return appropriate counters + * DAT_HANDLE dat_handle, enum cntr, *DAT_UINT64 p_cntrs_out, int reset + * + * use _ALL_COUNTERS to query all + */ +#define dat_query_counters(dat_handle, cntr, p_cntrs_out, reset) \ + dat_extension_op( dat_handle, \ + DAT_QUERY_COUNTERS_OP, \ + (cntr), \ + (p_cntrs_out), \ + (reset)) +/* + * Print counter(s): + * Provide IA, EP, or EVD and call will print appropriate counters + * DAT_HANDLE dat_handle, int cntr, int reset + * + * use _ALL_COUNTERS to print all + */ +#define dat_print_counters(dat_handle, cntr, reset) \ + dat_extension_op( dat_handle, \ + DAT_PRINT_COUNTERS_OP, \ + (cntr), \ + (reset)) + #endif /* _DAT_IB_EXTENSIONS_H_ */ diff --git a/test/dtest/dtestx.c b/test/dtest/dtestx.c index 032cadd..eafc1c9 100755 --- a/test/dtest/dtestx.c +++ b/test/dtest/dtestx.c @@ -60,6 +60,7 @@ #define DAPL_PROVIDER "ofa-v2-ib0" #define F64x "%"PRIx64"" +#define F64u "%"PRIu64"" #endif @@ -74,6 +75,7 @@ int disconnect_ep(void); if (status != DAT_SUCCESS) {\ dat_strerror(status, &maj_msg, &min_msg);\ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\ + dat_ia_close(ia, DAT_CLOSE_DEFAULT);\ exit(1);\ } else if (verbose) {\ printf("dtestx: %s success\n",str);\ @@ -86,6 +88,7 @@ int disconnect_ep(void); if (status != DAT_SUCCESS) {\ dat_strerror(status, &maj_msg, &min_msg);\ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\ + dat_ia_close(ia, DAT_CLOSE_DEFAULT);\ exit(1);\ } else if (verbose) {\ printf("dtestx: %s\n",str);\ @@ -162,6 +165,7 @@ DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL; DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL; DAT_PSP_HANDLE psp = DAT_HANDLE_NULL; int server = 1; +int remote_host = 0; int ud_test = 0; int multi_eps = 0; int buf_size = BUF_SIZE; @@ -171,6 +175,8 @@ char hostname[256] = { 0 }; DAT_IB_ADDR_HANDLE remote_ah[MAX_EP_COUNT]; int eps = 1; int verbose = 0; +int counters = 0; +int counters_ok = 0; #define LOGPRINTF if (verbose) printf @@ -178,11 +184,12 @@ void print_usage(void) { printf("\n dtestx usage \n\n"); printf("v: verbose\n"); + printf("p: print counters\n"); printf("u unreliable datagram test\n"); printf("U: unreliable datagram test, UD endpoint count\n"); printf("m unreliable datagram test, multiple Server endpoints\n"); printf("b: buf length to allocate\n"); - printf("h: hostname/address of Server, specified on Client\n"); + printf("h: hostname/address of Server, client and UDP server\n"); printf("c: Client\n"); printf("s: Server, default\n"); printf("P: provider name (default = ofa-v2-ib0)\n"); @@ -400,6 +407,7 @@ connect_ep(char *hostname) DAT_RMR_TRIPLET *r_iov; DAT_DTO_COOKIE cookie; DAT_CONN_QUAL conn_qual; + DAT_BOOLEAN in,out; int i,ii,pdata,ctx; DAT_PROVIDER_ATTR prov_attrs; DAT_DTO_COMPLETION_EVENT_DATA *dto_event = @@ -418,6 +426,19 @@ connect_ep(char *hostname) LOGPRINTF(" Provider Specific Attribute[%d] %s=%s\n", i, prov_attrs.provider_specific_attr[i].name, prov_attrs.provider_specific_attr[i].value); + + /* check for counter support */ + status = strcmp(prov_attrs.provider_specific_attr[i].name, + "DAT_COUNTERS"); + if (!status) + counters_ok = 1; + } + + /* make sure provider supports counters */ + if ((counters) && (!counters_ok)) { + printf("Disable dat_query_counters:" + " Provider not built with counters\n"); + counters = 0; } status = dat_pz_create(ia, &pz); @@ -632,6 +653,9 @@ connect_ep(char *hostname) cookie, DAT_COMPLETION_SUPPRESS_FLAG); + dat_ep_get_status(ep[0],NULL,&in,&out); + printf("EP[0] status: posted buffers: Req=%d, Rcv=%d\n",in,out); + /* * Wait for their RMR */ @@ -730,9 +754,49 @@ disconnect_ep(void) _OK2(status, "dat_lmr_free_atomic"); } for (i=0;i