dapl: endpoint pending request count is wrong

The code assumes every cookie allocated during posting of
requests gets completed. This incorrect assumption results in
wrong pending count. Remove request_pending field and replace
with direct call, dapl_cb_pending, to provide accurate
data to consumer.

Add debug print if consumer overruns request queue.

Signed-off by: Arlin Davis ardavis@ichips.intel.com

diff --git a/dapl/common/dapl_cookie.c b/dapl/common/dapl_cookie.c
index ee12ce20178cd40715bcc6592efc7b5c8464eddd..5b0b59823bfbcc3f837b6a16a6a337d2702a6838 100644 (file)
--- a/dapl/common/dapl_cookie.c
+++ b/dapl/common/dapl_cookie.c
@@ -260,6 +260,34 @@ dapls_cb_put (
     return DAT_SUCCESS;
 }
 
+/*
+ * dapls_cb_pending
+ *
+ * snapshot of active entries on cookie ring buffer 
+ *
+ * Input:
+ *     buffer          pointer to DAPL_COOKIE_BUFFER
+ *
+ * Returns:
+ *     DAT_COUNT       number of active/pending cookies
+ *
+ */
+extern DAT_COUNT 
+dapls_cb_pending (
+    DAPL_COOKIE_BUFFER *buffer )
+{
+       DAT_COUNT head, tail;
+
+       head = dapl_os_atomic_read(&buffer->head);
+       tail = dapl_os_atomic_read(&buffer->tail);
+
+       if (head == tail)
+               return 0;
+       else if (head > tail)
+               return (head - tail);
+       else 
+               return ((buffer->pool_size - tail) + head);
+}
 
 /*
  * dapls_rmr_cookie_alloc

diff --git a/dapl/common/dapl_cookie.h b/dapl/common/dapl_cookie.h
index 20fbec643880f4c0feeff3842741c5151f3f1c38..f953b28a30a79dc7c62381a8cf065b56a61a5052 100644 (file)
--- a/dapl/common/dapl_cookie.h
+++ b/dapl/common/dapl_cookie.h
@@ -50,6 +50,10 @@ extern void
 dapls_cb_free (
     DAPL_COOKIE_BUFFER         *buffer );
 
+extern DAT_COUNT 
+dapls_cb_pending (
+    DAPL_COOKIE_BUFFER         *buffer );
+
 extern DAT_RETURN
 dapls_rmr_cookie_alloc (
     IN  DAPL_COOKIE_BUFFER     *buffer,

diff --git a/dapl/common/dapl_ep_get_status.c b/dapl/common/dapl_ep_get_status.c
index 49b4fef444f3dc4c72a11c5d7206982e9e1e4b72..a9313553e335f11c81534386ac417653babe3596 100644 (file)
--- a/dapl/common/dapl_ep_get_status.c
+++ b/dapl/common/dapl_ep_get_status.c
@@ -98,12 +98,12 @@ dapl_ep_get_status (
 
     if ( in_dto_idle != NULL )
     {
-       *in_dto_idle = (dapl_os_atomic_read (&ep_ptr->recv_count)) ? DAT_FALSE : DAT_TRUE;
+       *in_dto_idle = (dapls_cb_pending(&ep_ptr->recv_buffer)) ? DAT_FALSE : DAT_TRUE;
     }
 
     if ( out_dto_idle != NULL )
     {
-       *out_dto_idle = (dapl_os_atomic_read (&ep_ptr->req_count)) ? DAT_FALSE : DAT_TRUE;
+       *out_dto_idle = (dapls_cb_pending(&ep_ptr->req_buffer)) ? DAT_FALSE : DAT_TRUE;
     }
 
  bail:

diff --git a/dapl/common/dapl_ep_post_recv.c b/dapl/common/dapl_ep_post_recv.c
index c9be9ec740a8a17576833d0a59deff5c8d43cbc4..5f903b8d28c8333c1e1cfab243df8add82de538b 100644 (file)
--- a/dapl/common/dapl_ep_post_recv.c
+++ b/dapl/common/dapl_ep_post_recv.c
@@ -108,12 +108,6 @@ dapl_ep_post_recv (
        goto bail;
     }
 
-    /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->recv_count);
-
     /*
      * Invoke provider specific routine to post DTO
      */
@@ -121,7 +115,6 @@ dapl_ep_post_recv (
 
     if ( dat_status != DAT_SUCCESS )
     {
-       dapl_os_atomic_dec (&ep_ptr->recv_count);
        dapls_cookie_dealloc (&ep_ptr->recv_buffer, cookie);
     }
 

diff --git a/dapl/common/dapl_ep_util.c b/dapl/common/dapl_ep_util.c
index d11aed338ee82de647ebee0504467651704e73d1..2bad346047d68ab09e8af3fc4a879e1a8efa9e13 100644 (file)
--- a/dapl/common/dapl_ep_util.c
+++ b/dapl/common/dapl_ep_util.c
@@ -131,9 +131,6 @@ dapl_ep_alloc (
     ep_ptr->qp_state  = DAPL_QP_STATE_UNATTACHED;
     ep_ptr->cm_handle = IB_INVALID_HANDLE;
 
-    dapl_os_atomic_set (&ep_ptr->req_count, 0);
-    dapl_os_atomic_set (&ep_ptr->recv_count, 0);
-
     if ( DAT_SUCCESS != dapls_cb_create (
             &ep_ptr->req_buffer,
             ep_ptr,
@@ -393,17 +390,19 @@ dapl_ep_post_send_req (
                 dto_type,
                 user_cookie,
                 &cookie );
-    if ( dat_status != DAT_SUCCESS )
-    {
+    if (dat_status != DAT_SUCCESS) {
+       dapl_log(DAPL_DBG_TYPE_ERR,
+                " dapl_post_req resource ERR:"
+                " dtos pending = %d, max_dtos %d, max_cb %d hd %d tl %d\n",
+                dapls_cb_pending(&ep_ptr->req_buffer),
+                ep_ptr->param.ep_attr.max_request_dtos,
+                ep_ptr->req_buffer.pool_size,
+                ep_ptr->req_buffer.head,
+                ep_ptr->req_buffer.tail);
+
        goto bail;
     }
 
-    /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     /*
      * Invoke provider specific routine to post DTO
      */
@@ -417,7 +416,6 @@ dapl_ep_post_send_req (
 
     if ( dat_status != DAT_SUCCESS )
     {
-       dapl_os_atomic_dec (&ep_ptr->req_count);
        dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
     }
 

diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c
index 2c95c6d9726c9c6636eb50d1db3175342e9b702a..41c128cc8db62548a5464ff6f16ff5844dd66593 100644 (file)
--- a/dapl/common/dapl_evd_util.c
+++ b/dapl/common/dapl_evd_util.c
@@ -1029,17 +1029,11 @@ dapli_evd_cqe_to_event (
        {
            DAPL_COOKIE_BUFFER  *buffer;
 
-           if ( DAPL_DTO_TYPE_RECV == cookie->val.dto.type )
-           {
-               dapl_os_atomic_dec (&ep_ptr->recv_count);
-               buffer = &ep_ptr->recv_buffer;
-           }
+           if (DAPL_DTO_TYPE_RECV == cookie->val.dto.type)
+               buffer = &ep_ptr->recv_buffer;
            else
-           {
-               dapl_os_atomic_dec (&ep_ptr->req_count);
-               buffer = &ep_ptr->req_buffer;
-           }
-
+               buffer = &ep_ptr->req_buffer;
+           
            event_ptr->event_number = DAT_DTO_COMPLETION_EVENT;
            event_ptr->event_data.dto_completion_event_data.ep_handle =
                cookie->ep;
@@ -1084,8 +1078,6 @@ dapli_evd_cqe_to_event (
 
        case DAPL_COOKIE_TYPE_RMR:
        {
-           dapl_os_atomic_dec (&ep_ptr->req_count);
-
            event_ptr->event_number = DAT_RMR_BIND_COMPLETION_EVENT;
 
            event_ptr->event_data.rmr_completion_event_data.rmr_handle =

diff --git a/dapl/common/dapl_rmr_bind.c b/dapl/common/dapl_rmr_bind.c
index 1582fb33c3c523800fc5fc833527ea2599e26026..905ea2c8e52ed1a1a6a8b6c7bdd94d9d70c5e654 100644 (file)
--- a/dapl/common/dapl_rmr_bind.c
+++ b/dapl/common/dapl_rmr_bind.c
@@ -151,12 +151,6 @@ dapli_rmr_bind_fuse (
 
     is_signaled = (completion_flags & DAT_COMPLETION_SUPPRESS_FLAG) ? DAT_FALSE : DAT_TRUE;
 
-    /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     dat_status = dapls_ib_mw_bind (rmr,
                                   lmr,
                                   ep_ptr,
@@ -167,7 +161,6 @@ dapli_rmr_bind_fuse (
                                   is_signaled);
     if ( DAT_SUCCESS != dat_status )
     {
-       dapl_os_atomic_dec (&ep_ptr->req_count);
        dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
        goto bail;
     }
@@ -255,19 +248,12 @@ dapli_rmr_bind_unfuse (
 
     is_signaled = (completion_flags & DAT_COMPLETION_UNSIGNALLED_FLAG) ? DAT_FALSE : DAT_TRUE;
 
-    /*
-     * Take reference before posting to avoid race conditions with 
-     * completions 
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     dat_status = dapls_ib_mw_unbind (rmr,
                                     ep_ptr,
                                     cookie,
                                     is_signaled);
     if ( DAT_SUCCESS != dat_status )
     {
-       dapl_os_atomic_dec (&ep_ptr->req_count);
        dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
        goto bail1;
     }