dapl: endpoint pending request count is wrong

The code assumes every cookie allocated during posting of
requests gets completed. This incorrect assumption results in
wrong pending count. Remove request_pending field and replace
with direct call, dapl_cb_pending, to provide accurate
data to consumer.

Add debug print if consumer overruns request queue.

Signed-off by: Arlin Davis ardavis@ichips.intel.com

diff --git a/dapl/common/dapl_cookie.c b/dapl/common/dapl_cookie.c
index 7998d7fce14342cd4328114d665236ce85838094..1462fe99890ca978e474af27b8d81fb5a84d2343 100644 (file)
--- a/dapl/common/dapl_cookie.c
+++ b/dapl/common/dapl_cookie.c
@@ -260,6 +260,34 @@ dapls_cb_put (
     return DAT_SUCCESS;
 }
 
+/*
+ * dapls_cb_pending
+ *
+ * snapshot of active entries on cookie ring buffer 
+ *
+ * Input:
+ *     buffer          pointer to DAPL_COOKIE_BUFFER
+ *
+ * Returns:
+ *     DAT_COUNT       number of active/pending cookies
+ *
+ */
+extern DAT_COUNT 
+dapls_cb_pending (
+    DAPL_COOKIE_BUFFER *buffer )
+{
+       DAT_COUNT head, tail;
+
+       head = dapl_os_atomic_read(&buffer->head);
+       tail = dapl_os_atomic_read(&buffer->tail);
+
+       if (head == tail)
+               return 0;
+       else if (head > tail)
+               return (head - tail);
+       else 
+               return ((buffer->pool_size - tail) + head);
+}
 
 /*
  * dapls_rmr_cookie_alloc

diff --git a/dapl/common/dapl_cookie.h b/dapl/common/dapl_cookie.h
index 20fbec643880f4c0feeff3842741c5151f3f1c38..f953b28a30a79dc7c62381a8cf065b56a61a5052 100644 (file)
--- a/dapl/common/dapl_cookie.h
+++ b/dapl/common/dapl_cookie.h
@@ -50,6 +50,10 @@ extern void
 dapls_cb_free (
     DAPL_COOKIE_BUFFER         *buffer );
 
+extern DAT_COUNT 
+dapls_cb_pending (
+    DAPL_COOKIE_BUFFER         *buffer );
+
 extern DAT_RETURN
 dapls_rmr_cookie_alloc (
     IN  DAPL_COOKIE_BUFFER     *buffer,

diff --git a/dapl/common/dapl_ep_get_status.c b/dapl/common/dapl_ep_get_status.c
index d55b51241964b944a05bbe121d9fba3ef5179d0d..853afff6dc405b6998fcc2a2bd4b3b96ec76160b 100644 (file)
--- a/dapl/common/dapl_ep_get_status.c
+++ b/dapl/common/dapl_ep_get_status.c
@@ -98,12 +98,12 @@ dapl_ep_get_status (
 
     if ( in_dto_idle != NULL )
     {
-       *in_dto_idle = (dapl_os_atomic_read (&ep_ptr->recv_count)) ? DAT_FALSE : DAT_TRUE;
+       *in_dto_idle = (dapls_cb_pending(&ep_ptr->recv_buffer)) ? DAT_FALSE : DAT_TRUE;
     }
 
     if ( out_dto_idle != NULL )
     {
-       *out_dto_idle = (dapl_os_atomic_read (&ep_ptr->req_count)) ? DAT_FALSE : DAT_TRUE;
+       *out_dto_idle = (dapls_cb_pending(&ep_ptr->req_buffer)) ? DAT_FALSE : DAT_TRUE;
     }
 
  bail:

diff --git a/dapl/common/dapl_ep_post_recv.c b/dapl/common/dapl_ep_post_recv.c
index 2e52411f19211f4ff382e4d6054f7a7c8bf6ecff..12724cddc7fc80a33bf3ac8f4c6d969f3326a4d3 100644 (file)
--- a/dapl/common/dapl_ep_post_recv.c
+++ b/dapl/common/dapl_ep_post_recv.c
@@ -108,12 +108,6 @@ dapl_ep_post_recv (
        goto bail;
     }
 
-    /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->recv_count);
-
     /*
      * Invoke provider specific routine to post DTO
      */
@@ -121,7 +115,6 @@ dapl_ep_post_recv (
 
     if ( dat_status != DAT_SUCCESS )
     {
-       dapl_os_atomic_dec (&ep_ptr->recv_count);
        dapls_cookie_dealloc (&ep_ptr->recv_buffer, cookie);
     }
 

diff --git a/dapl/common/dapl_ep_util.c b/dapl/common/dapl_ep_util.c
index e29e2872c1a2d8955b478d4f813859fd3c7f1cbe..f3f88bc845a17c0254916ecf09aeba242f334aa8 100644 (file)
--- a/dapl/common/dapl_ep_util.c
+++ b/dapl/common/dapl_ep_util.c
@@ -161,9 +161,6 @@ dapl_ep_alloc (
     ep_ptr->qp_state  = DAPL_QP_STATE_UNATTACHED;
     ep_ptr->cm_handle = IB_INVALID_HANDLE;
 
-    dapl_os_atomic_set (&ep_ptr->req_count, 0);
-    dapl_os_atomic_set (&ep_ptr->recv_count, 0);
-
     if ( DAT_SUCCESS != dapls_cb_create (
             &ep_ptr->req_buffer,
             ep_ptr,
@@ -431,17 +428,19 @@ dapl_ep_post_send_req (
                 dto_type,
                 user_cookie,
                 &cookie );
-    if ( dat_status != DAT_SUCCESS )
-    {
+    if (dat_status != DAT_SUCCESS) {
+       dapl_log(DAPL_DBG_TYPE_ERR,
+                " dapl_post_req resource ERR:"
+                " dtos pending = %d, max_dtos %d, max_cb %d hd %d tl %d\n",
+                dapls_cb_pending(&ep_ptr->req_buffer),
+                ep_ptr->param.ep_attr.max_request_dtos,
+                ep_ptr->req_buffer.pool_size,
+                ep_ptr->req_buffer.head,
+                ep_ptr->req_buffer.tail);
+
        goto bail;
     }
 
-    /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     /*
      * Invoke provider specific routine to post DTO
      */
@@ -455,7 +454,6 @@ dapl_ep_post_send_req (
 
     if ( dat_status != DAT_SUCCESS )
     {
-       dapl_os_atomic_dec (&ep_ptr->req_count);
        dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
     }
 

diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c
index 293759fa64b6c8e7e46615535f4fc28b3807958f..87a0194d54c1b10a6cb47ee45247a0ecddf462b5 100755 (executable)
--- a/dapl/common/dapl_evd_util.c
+++ b/dapl/common/dapl_evd_util.c
@@ -1099,17 +1099,11 @@ dapli_evd_cqe_to_event (
            }
 #endif
 
-           if ( DAPL_DTO_TYPE_RECV == cookie->val.dto.type )
-           {
-               dapl_os_atomic_dec (&ep_ptr->recv_count);
-               buffer = &ep_ptr->recv_buffer;
-           }
+           if (DAPL_DTO_TYPE_RECV == cookie->val.dto.type)
+               buffer = &ep_ptr->recv_buffer;
            else
-           {
-               dapl_os_atomic_dec (&ep_ptr->req_count);
-               buffer = &ep_ptr->req_buffer;
-           }
-
+               buffer = &ep_ptr->req_buffer;
+           
            event_ptr->event_number = DAT_DTO_COMPLETION_EVENT;
            event_ptr->event_data.dto_completion_event_data.ep_handle =
                cookie->ep;
@@ -1136,8 +1130,6 @@ dapli_evd_cqe_to_event (
 
        case DAPL_COOKIE_TYPE_RMR:
        {
-           dapl_os_atomic_dec (&ep_ptr->req_count);
-
            event_ptr->event_number = DAT_RMR_BIND_COMPLETION_EVENT;
 
            event_ptr->event_data.rmr_completion_event_data.rmr_handle =

diff --git a/dapl/common/dapl_rmr_bind.c b/dapl/common/dapl_rmr_bind.c
index 01d06c37046ca7ca776fdeb2c774ee279966ffe1..12a98f6fea1dd097b34e0212466a59db59c05aab 100755 (executable)
--- a/dapl/common/dapl_rmr_bind.c
+++ b/dapl/common/dapl_rmr_bind.c
@@ -151,12 +151,6 @@ dapli_rmr_bind_fuse (
 
     is_signaled = (completion_flags & DAT_COMPLETION_SUPPRESS_FLAG) ? DAT_FALSE : DAT_TRUE;
 
-    /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     dat_status = dapls_ib_mw_bind (rmr,
                                   lmr,
                                   ep_ptr,
@@ -167,7 +161,6 @@ dapli_rmr_bind_fuse (
                                   is_signaled);
     if ( DAT_SUCCESS != dat_status )
     {
-       dapl_os_atomic_dec (&ep_ptr->req_count);
        dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
        goto bail;
     }
@@ -255,19 +248,12 @@ dapli_rmr_bind_unfuse (
 
     is_signaled = (completion_flags & DAT_COMPLETION_UNSIGNALLED_FLAG) ? DAT_FALSE : DAT_TRUE;
 
-    /*
-     * Take reference before posting to avoid race conditions with 
-     * completions 
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     dat_status = dapls_ib_mw_unbind (rmr,
                                     ep_ptr,
                                     cookie,
                                     is_signaled);
     if ( DAT_SUCCESS != dat_status )
     {
-       dapl_os_atomic_dec (&ep_ptr->req_count);
        dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
        goto bail1;
     }