ucm,scm: UD mode creates many CR objects per EP that needs cleaned up

After connection is established and the AH is provided to consumer
on UD connect establishment there is no need to keep the CR object
on the SP. For large clusters this results in a growing memory
footprint for CR objects and long cleanup times on device close.

Change ucm and scm providers to unlink and free CR resources
during CM object free if this is a UD QP and CONN_EST state.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/common/dapl_sp_util.c b/dapl/common/dapl_sp_util.c
index fba371137d21d523a34eabeb375dee911db37d49..27bc3069db1a22815ceec8c540d053f02198596c 100644 (file)
--- a/dapl/common/dapl_sp_util.c
+++ b/dapl/common/dapl_sp_util.c
@@ -226,6 +226,9 @@ void dapl_sp_remove_cr(IN DAPL_SP * sp_ptr, IN DAPL_CR * cr_ptr)
                return;
        }
 
+       dapl_log(DAPL_DBG_TYPE_CM, " dapl_sp_remove_cr: SP %p CR %p cnt %d\n",
+               sp_ptr, cr_ptr, sp_ptr->cr_list_count);
+
        dapl_llist_remove_entry(&sp_ptr->cr_list_head,
                                &cr_ptr->header.ia_list_entry);
        sp_ptr->cr_list_count--;

diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index 514944985d3c5ba735b114be4f2a3214722458d2..a47711480d08a7c8654179ace04624807da977e4 100644 (file)
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -58,6 +58,7 @@
 #include "dapl_name_service.h"
 #include "dapl_ib_util.h"
 #include "dapl_ep_util.h"
+#include "dapl_sp_util.h"
 #include "dapl_osd.h"
 
 /* forward declarations */
@@ -428,13 +429,26 @@ static void dapli_cm_dequeue(dp_ib_cm_handle_t cm_ptr)
        dapls_cm_release(cm_ptr);
 }
 
-/* BLOCKING: called from dapl_ep_free, EP link will be last ref */
+/* BLOCKING: called from dapl_ep_free, EP link will be last ref, cleanup UD CR */
 void dapls_cm_free(dp_ib_cm_handle_t cm_ptr)
 {
+       DAPL_SP *sp_ptr = cm_ptr->sp;
+
        dapl_log(DAPL_DBG_TYPE_CM,
-                " cm_free: cm %p %s ep %p refs=%d\n", 
+                " cm_free: cm %p %s ep %p sp %p refs=%d\n",
                 cm_ptr, dapl_cm_state_str(cm_ptr->state),
-                cm_ptr->ep, cm_ptr->ref_count);
+                cm_ptr->ep, sp_ptr, cm_ptr->ref_count);
+
+       if (sp_ptr && cm_ptr->state == DCM_CONNECTED &&
+           cm_ptr->msg.daddr.ib.qp_type == IBV_QPT_UD) {
+               DAPL_CR *cr_ptr = dapl_sp_search_cr(sp_ptr, cm_ptr);
+               if (cr_ptr != NULL) {
+                       dapl_os_lock(&sp_ptr->header.lock);
+                       dapl_sp_remove_cr(sp_ptr, cr_ptr);
+                       dapl_os_unlock(&sp_ptr->header.lock);
+                       dapls_cr_free(cr_ptr);
+               }
+       }
        
        /* free from internal workq, wait until EP is last ref */
        dapl_os_lock(&cm_ptr->lock);

diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index b4a7c223389b91c627c42cd191768f7b98021c63..b6d1fc5348b1f9bf97fbd4613ae1a29a7424e69b 100644 (file)
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -32,6 +32,7 @@
 #include "dapl_name_service.h"
 #include "dapl_ib_util.h"
 #include "dapl_ep_util.h"
+#include "dapl_sp_util.h"
 #include "dapl_osd.h"
 
 
@@ -768,13 +769,27 @@ bail:
        return NULL;
 }
 
-/* schedule destruction of CM object */
+/* schedule destruction of CM object, clean UD CR */
 void dapli_cm_free(dp_ib_cm_handle_t cm)
 {
+       DAPL_SP *sp_ptr = cm->sp;
+
        dapl_log(DAPL_DBG_TYPE_CM,
-                " dapli_cm_free: cm %p %s ep %p refs=%d\n", 
+                " dapli_cm_free: cm %p %s ep %p sp %p cr_cnt %d refs=%d\n",
                 cm, dapl_cm_state_str(cm->state),
-                cm->ep, cm->ref_count);
+                cm->ep, sp_ptr, sp_ptr ? sp_ptr->cr_list_count:0, cm->ref_count);
+
+       if (sp_ptr && cm->state == DCM_CONNECTED &&
+           cm->msg.daddr.ib.qp_type == IBV_QPT_UD) {
+               DAPL_CR *cr_ptr = dapl_sp_search_cr(sp_ptr, cm);
+               dapl_log(DAPL_DBG_TYPE_CM, " dapli_cm_free: UD CR %p\n", cr_ptr);
+               if (cr_ptr != NULL) {
+                       dapl_os_lock(&sp_ptr->header.lock);
+                       dapl_sp_remove_cr(sp_ptr, cr_ptr);
+                       dapl_os_unlock(&sp_ptr->header.lock);
+                       dapls_cr_free(cr_ptr);
+               }
+       }
 
        dapl_os_lock(&cm->lock);
        cm->state = DCM_FREE;