From 615619baa0e8be70fd30a890bac007247759802b Mon Sep 17 00:00:00 2001
From: Sean Hefty <sean.hefty@intel.com>
Date: Thu, 14 Oct 2010 10:46:46 -0700
Subject: [PATCH] dapl: fix spawn test

---
 trunk/ulp/dapl2/dapl/common/dapl_ep_free.c | 35 +++++++++++++++-------
 trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c   | 15 ++++++++--
 trunk/ulp/dapl2/dapl/openib_common/qp.c    | 32 ++++++++++++--------
 3 files changed, 56 insertions(+), 26 deletions(-)

diff --git a/trunk/ulp/dapl2/dapl/common/dapl_ep_free.c b/trunk/ulp/dapl2/dapl/common/dapl_ep_free.c
index 32d50cce..56b36869 100644
--- a/trunk/ulp/dapl2/dapl/common/dapl_ep_free.c
+++ b/trunk/ulp/dapl2/dapl/common/dapl_ep_free.c
@@ -157,16 +157,16 @@ DAT_RETURN DAT_API dapl_ep_free(IN DAT_EP_HANDLE ep_handle)
 				   pz_ref_count);
 		param->pz_handle = NULL;
 	}
-	if (param->recv_evd_handle != NULL) {
-		dapl_os_atomic_dec(&((DAPL_EVD *) param->recv_evd_handle)->
-				   evd_ref_count);
-		param->recv_evd_handle = NULL;
-	}
-	if (param->request_evd_handle != NULL) {
-		dapl_os_atomic_dec(&((DAPL_EVD *) param->request_evd_handle)->
-				   evd_ref_count);
-		param->request_evd_handle = NULL;
-	}
+	//if (param->recv_evd_handle != NULL) {
+	//	dapl_os_atomic_dec(&((DAPL_EVD *) param->recv_evd_handle)->
+	//			   evd_ref_count);
+	//	param->recv_evd_handle = NULL;
+	//}
+	//if (param->request_evd_handle != NULL) {
+	//	dapl_os_atomic_dec(&((DAPL_EVD *) param->request_evd_handle)->
+	//			   evd_ref_count);
+	//	param->request_evd_handle = NULL;
+	//}
 	if (param->connect_evd_handle != NULL) {
 		dapl_os_atomic_dec(&((DAPL_EVD *) param->connect_evd_handle)->
 				   evd_ref_count);
@@ -202,7 +202,20 @@ DAT_RETURN DAT_API dapl_ep_free(IN DAT_EP_HANDLE ep_handle)
 		}
 	}
 
-	dapls_ep_flush_cqs(ep_ptr);
+	/*
+	 * Release the EVD handles after we destroy the QP, so we can flush all
+	 * QP entries.
+	 */
+	if (param->recv_evd_handle != NULL) {
+		dapl_os_atomic_dec(&((DAPL_EVD *) param->recv_evd_handle)->
+				   evd_ref_count);
+		param->recv_evd_handle = NULL;
+	}
+	if (param->request_evd_handle != NULL) {
+		dapl_os_atomic_dec(&((DAPL_EVD *) param->request_evd_handle)->
+				   evd_ref_count);
+		param->request_evd_handle = NULL;
+	}
 
 	/* Free the resource */
 	dapl_ep_dealloc(ep_ptr);
diff --git a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c
index e843829d..7a64f1d1 100644
--- a/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c
+++ b/trunk/ulp/dapl2/dapl/ibal/dapl_ibal_qp.c
@@ -317,6 +317,7 @@ dapls_ib_qp_free (
         IN  DAPL_IA                *ia_ptr,
         IN  DAPL_EP                *ep_ptr )
 {
+	ib_qp_handle_t qp;
 
 	UNREFERENCED_PARAMETER(ia_ptr);
 
@@ -327,12 +328,20 @@ dapls_ib_qp_free (
 	dapl_os_lock(&ep_ptr->header.lock);
 	if (( ep_ptr->qp_handle != IB_INVALID_HANDLE ))
 	{
-		ib_destroy_qp ( ep_ptr->qp_handle, ib_sync_destroy );
+		qp  = ep_ptr->qp_handle;
+		ep_ptr->qp_handle = IB_INVALID_HANDLE;
+		dapl_os_unlock(&ep_ptr->header.lock);
+
+		dapls_modify_qp_state_to_error(qp);
+		dapls_modify_qp_state_to_reset(qp);
+		dapls_ep_flush_cqs(ep_ptr);
+
+		ib_destroy_qp ( qp, ib_sync_destroy );
 		dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsQF: freed QP %p\n",
 				ep_ptr->qp_handle ); 
-		ep_ptr->qp_handle = IB_INVALID_HANDLE;
+	} else {
+		dapl_os_unlock(&ep_ptr->header.lock);
 	}
-	dapl_os_unlock(&ep_ptr->header.lock);
 
     return DAT_SUCCESS;
 }
diff --git a/trunk/ulp/dapl2/dapl/openib_common/qp.c b/trunk/ulp/dapl2/dapl/openib_common/qp.c
index 179eef0e..6024db3d 100644
--- a/trunk/ulp/dapl2/dapl/openib_common/qp.c
+++ b/trunk/ulp/dapl2/dapl/openib_common/qp.c
@@ -209,30 +209,38 @@ dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,
  */
 DAT_RETURN dapls_ib_qp_free(IN DAPL_IA * ia_ptr, IN DAPL_EP * ep_ptr)
 {
+	struct ibv_qp *qp;
+
 #ifdef _OPENIB_CMA_
 	dp_ib_cm_handle_t cm_ptr = dapl_get_cm_from_ep(ep_ptr);
+	if (!cm_ptr)
+		return DAT_SUCCESS;
+#endif
 
-	dapl_os_lock(&ep_ptr->header.lock);
-	if (cm_ptr && cm_ptr->cm_id->qp) {
-		rdma_destroy_qp(cm_ptr->cm_id);
-		cm_ptr->cm_id->qp = NULL;
-		ep_ptr->qp_handle = NULL;
-	}
-#else
 	dapl_os_lock(&ep_ptr->header.lock);
 	if (ep_ptr->qp_handle != NULL) {
+		qp = ep_ptr->qp_handle;
+		ep_ptr->qp_handle = NULL;
+		dapl_os_unlock(&ep_ptr->header.lock);
+
 		/* force error state to flush queue, then destroy */
-		dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_ERR, 0,0,0);
+		dapls_modify_qp_state(qp, IBV_QPS_ERR, 0,0,0);
+		dapls_modify_qp_state(qp, IBV_QPS_RESET, 0,0,0);
+		dapls_ep_flush_cqs(ep_ptr);
 
-		if (ibv_destroy_qp(ep_ptr->qp_handle)) {
+#ifdef _OPENIB_CMA_
+		rdma_destroy_qp(cm_ptr->cm_id);
+		cm_ptr->cm_id->qp = NULL;
+#else
+		if (ibv_destroy_qp(qp)) {
 			dapl_log(DAPL_DBG_TYPE_ERR, 
 				 " qp_free: ibv_destroy_qp error - %s\n",
 				 strerror(errno));
 		}
-		ep_ptr->qp_handle = NULL;
-	}
 #endif
-	dapl_os_unlock(&ep_ptr->header.lock);
+	} else {
+		dapl_os_unlock(&ep_ptr->header.lock);
+	}
 	return DAT_SUCCESS;
 }
 
-- 
2.46.0