ucm: tighten up locking with CM processing, state changes

tighten up locking on CM processing and state changes
and reduce the send completion threshold to 50 from 100
to replenish the request message faster.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index 5c5287fd8b66ffc1391b3273858b583c810e2f76..e76e920f412ce50720ec5a297d0b45e2abd3a886 100644 (file)
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -185,7 +185,7 @@ static void ucm_accept_rtu(dp_ib_cm_handle_t cm, ib_cm_msg_t *msg);
 static int ucm_send(ib_hca_transport_t *tp, ib_cm_msg_t *msg, DAT_PVOID p_data, DAT_COUNT p_size);
 DAT_RETURN dapli_cm_disconnect(dp_ib_cm_handle_t cm);
 
-#define UCM_SND_BURST  100
+#define UCM_SND_BURST  50      
 
 /* Service ids - port space */
 static uint16_t ucm_get_port(ib_hca_transport_t *tp, uint16_t port)
@@ -916,11 +916,14 @@ static void ucm_connect_rtu(dp_ib_cm_handle_t cm, ib_cm_msg_t *msg)
        /* Send RTU, no private data */
        cm->msg.op = htons(DCM_RTU);
        
+       dapl_os_lock(&cm->lock);
+       cm->state = DCM_CONNECTED;
+       dapl_os_unlock(&cm->lock);
+
        if (ucm_send(&cm->hca->ib_trans, &cm->msg, NULL, 0))            
                goto bail;
 
        /* init cm_handle and post the event with private data */
-       cm->state = DCM_CONNECTED;
        dapl_dbg_log(DAPL_DBG_TYPE_EP, " ACTIVE: connected!\n");
 
 #ifdef DAT_EXTENSIONS
@@ -986,7 +989,10 @@ ud_bail:
                                (DAT_PVOID *)&xevent);
 
                /* we are done, don't destroy cm_ptr, need pdata */
+               dapl_os_lock(&cm->lock);
                cm->state = DCM_RELEASED;
+               dapl_os_unlock(&cm->lock);
+               
        } else
 #endif
        {
@@ -1157,7 +1163,9 @@ static void ucm_accept_rtu(dp_ib_cm_handle_t cm, ib_cm_msg_t *msg)
                                (DAT_PVOID *)&xevent);
 
                 /* done with CM object, don't destroy cm, need pdata */
+               dapl_os_lock(&cm->lock);
                        cm->state = DCM_RELEASED;
+               dapl_os_unlock(&cm->lock);
        } else {
 #endif
                cm->ep->cm_handle = cm; /* only RC, multi CR's on UD */
@@ -1262,8 +1270,6 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
         */
        cm->p_size = p_size;
        dapl_os_memcpy(&cm->p_data, p_data, p_size);
-       if (ucm_send(&cm->hca->ib_trans, &cm->msg, p_data, p_size))             
-               goto bail;
 
        /* save state and setup valid reference to EP, HCA */
        dapl_os_lock(&cm->lock);
@@ -1272,6 +1278,9 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
        cm->state = DCM_ACCEPTED;
        dapl_os_unlock(&cm->lock);
 
+       if (ucm_send(&cm->hca->ib_trans, &cm->msg, p_data, p_size))             
+               goto bail;
+
        dapl_dbg_log(DAPL_DBG_TYPE_CM, " PASSIVE: accepted!\n");
        return DAT_SUCCESS;
 
@@ -1587,14 +1596,17 @@ dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm,
        if (psize)
                dapl_os_memcpy(&cm->msg.p_data, pdata, psize);
                
+       /* cr_thread will destroy CR */
+       dapl_os_lock(&cm->lock);
+       cm->state = DCM_REJECTING;
+       dapl_os_unlock(&cm->lock);
+
        if (ucm_send(&cm->hca->ib_trans, &cm->msg, NULL, 0)) {
                dapl_log(DAPL_DBG_TYPE_WARN,
                         " cm_reject: ERR: %s\n", strerror(errno));
                return DAT_INTERNAL_ERROR;
        }
                
-       /* cr_thread will destroy CR */
-       cm->state = DCM_REJECTING;
        send(cm->hca->ib_trans.scm[1], "w", sizeof "w", 0);
        return DAT_SUCCESS;
 }