dapl scm: fix ibv_destroy_cq busy error condition during dat_evd_free.

Problem surfaced while running Intel MPI 3.1 and mpich2-test suite.
dapli_destroy_cm was incorrectly removing reference to qp_handle in endpoint
and destroy_cq and destroy_pd code was ignoring verbs errors.

Signed-off by: Arlin Davis ardavis@ichips.intel.com

diff --git a/dapl/openib_scm/dapl_ib_cm.c b/dapl/openib_scm/dapl_ib_cm.c
index 9e686d632fd0e5a1d205c7cb1cdfa057ab1bdfab..f78ebe666c05c8c286707461e5e850bb2cc813d9 100644 (file)
--- a/dapl/openib_scm/dapl_ib_cm.c
+++ b/dapl/openib_scm/dapl_ib_cm.c
@@ -108,10 +108,8 @@ static void dapli_cm_destroy(struct ib_cm_handle *cm_ptr)
 
        dapl_os_lock(&cm_ptr->lock);
        cm_ptr->state = SCM_DESTROY;
-       if (cm_ptr->ep) {
+       if (cm_ptr->ep) 
                cm_ptr->ep->cm_handle = IB_INVALID_HANDLE;
-               cm_ptr->ep->qp_handle = IB_INVALID_HANDLE;
-       }
 
        /* close socket if still active */
        if (cm_ptr->socket >= 0) {

diff --git a/dapl/openib_scm/dapl_ib_cq.c b/dapl/openib_scm/dapl_ib_cq.c
index 56b729e1732c04e180134467d8ac46d101729be9..7d6bd4ff9e7b9bc5e9e089f3c472fad3a84a5bf4 100644 (file)
--- a/dapl/openib_scm/dapl_ib_cq.c
+++ b/dapl/openib_scm/dapl_ib_cq.c
@@ -423,20 +423,15 @@ DAT_RETURN dapls_ib_cq_free (
        IN  DAPL_EVD            *evd_ptr)
 {
        DAT_EVENT event;
+       ib_work_completion_t wc;        
 
        if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) {
                /* pull off CQ and EVD entries and toss */      
-               while (dapls_ib_completion_poll == DAT_SUCCESS);
-               while (dapl_evd_dequeue(evd_ptr,&event) != DAT_QUEUE_EMPTY);
-#if 1 
-               ibv_destroy_cq(evd_ptr->ib_cq_handle); 
-               evd_ptr->ib_cq_handle = IB_INVALID_HANDLE;
-               return DAT_SUCCESS;
-#else
+               while (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, &wc) == 1);
+               while (dapl_evd_dequeue(evd_ptr, &event) == DAT_SUCCESS);
                if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) 
-                       return(dapl_convert_errno(errno,"destroy_cq"));
+                       return(dapl_convert_errno(errno,"ibv_destroy_cq"));
                evd_ptr->ib_cq_handle = IB_INVALID_HANDLE;
-#endif
        }
        return DAT_SUCCESS;
 }

diff --git a/dapl/openib_scm/dapl_ib_mem.c b/dapl/openib_scm/dapl_ib_mem.c
index 6a5e4a214abc5393f9ec46c63ba1bd6f26e26ac7..0faccc814f5ac615250074ce19defebb52a1f70f 100644 (file)
--- a/dapl/openib_scm/dapl_ib_mem.c
+++ b/dapl/openib_scm/dapl_ib_mem.c
@@ -136,15 +136,9 @@ DAT_RETURN
 dapls_ib_pd_free(IN DAPL_PZ *pz )
 {
        if (pz->pd_handle != IB_INVALID_HANDLE) {
-#if 1
-               ibv_dealloc_pd(pz->pd_handle);
-               pz->pd_handle = IB_INVALID_HANDLE;      
-               return DAT_SUCCESS;
-#else
                if (ibv_dealloc_pd(pz->pd_handle))
-                       return(dapl_convert_errno(errno,"dealloc_pd"));
+                       return(dapl_convert_errno(errno,"ibv_dealloc_pd"));
                pz->pd_handle = IB_INVALID_HANDLE;      
-#endif
        }
        return DAT_SUCCESS;
 }