From 617952fec1b8ca5ec5fc96d5568b90c1ba833ca3 Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Fri, 12 Jul 2013 11:52:33 -0700 Subject: [PATCH] ucm,scm: UD mode creates many CR objects per EP that needs cleaned up After connection is established and the AH is provided to consumer on UD connect establishment there is no need to keep the CR object on the SP. For large clusters this results in a growing memory footprint for CR objects and long cleanup times on device close. Change ucm and scm providers to unlink and free CR resources during CM object free if this is a UD QP and CONN_EST state. Signed-off-by: Arlin Davis --- dapl/common/dapl_sp_util.c | 3 +++ dapl/openib_scm/cm.c | 20 +++++++++++++++++--- dapl/openib_ucm/cm.c | 21 ++++++++++++++++++--- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/dapl/common/dapl_sp_util.c b/dapl/common/dapl_sp_util.c index fba3711..27bc306 100644 --- a/dapl/common/dapl_sp_util.c +++ b/dapl/common/dapl_sp_util.c @@ -226,6 +226,9 @@ void dapl_sp_remove_cr(IN DAPL_SP * sp_ptr, IN DAPL_CR * cr_ptr) return; } + dapl_log(DAPL_DBG_TYPE_CM, " dapl_sp_remove_cr: SP %p CR %p cnt %d\n", + sp_ptr, cr_ptr, sp_ptr->cr_list_count); + dapl_llist_remove_entry(&sp_ptr->cr_list_head, &cr_ptr->header.ia_list_entry); sp_ptr->cr_list_count--; diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c index b095c2f..d4964bd 100644 --- a/dapl/openib_scm/cm.c +++ b/dapl/openib_scm/cm.c @@ -58,6 +58,7 @@ #include "dapl_name_service.h" #include "dapl_ib_util.h" #include "dapl_ep_util.h" +#include "dapl_sp_util.h" #include "dapl_osd.h" /* forward declarations */ @@ -428,13 +429,26 @@ static void dapli_cm_dequeue(dp_ib_cm_handle_t cm_ptr) dapls_cm_release(cm_ptr); } -/* BLOCKING: called from dapl_ep_free, EP link will be last ref */ +/* BLOCKING: called from dapl_ep_free, EP link will be last ref, cleanup UD CR */ void dapls_cm_free(dp_ib_cm_handle_t cm_ptr) { + DAPL_SP *sp_ptr = cm_ptr->sp; + dapl_log(DAPL_DBG_TYPE_CM, - " cm_free: cm %p %s ep %p refs=%d\n", + " cm_free: cm %p %s ep %p sp %p refs=%d\n", cm_ptr, dapl_cm_state_str(cm_ptr->state), - cm_ptr->ep, cm_ptr->ref_count); + cm_ptr->ep, sp_ptr, cm_ptr->ref_count); + + if (sp_ptr && cm_ptr->state == DCM_CONNECTED && + cm_ptr->msg.daddr.ib.qp_type == IBV_QPT_UD) { + DAPL_CR *cr_ptr = dapl_sp_search_cr(sp_ptr, cm_ptr); + if (cr_ptr != NULL) { + dapl_os_lock(&sp_ptr->header.lock); + dapl_sp_remove_cr(sp_ptr, cr_ptr); + dapl_os_unlock(&sp_ptr->header.lock); + dapls_cr_free(cr_ptr); + } + } /* free from internal workq, wait until EP is last ref */ dapl_os_lock(&cm_ptr->lock); diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c index 4e6c527..05cff10 100644 --- a/dapl/openib_ucm/cm.c +++ b/dapl/openib_ucm/cm.c @@ -32,6 +32,7 @@ #include "dapl_name_service.h" #include "dapl_ib_util.h" #include "dapl_ep_util.h" +#include "dapl_sp_util.h" #include "dapl_osd.h" @@ -768,13 +769,27 @@ bail: return NULL; } -/* schedule destruction of CM object */ +/* schedule destruction of CM object, clean UD CR */ void dapli_cm_free(dp_ib_cm_handle_t cm) { + DAPL_SP *sp_ptr = cm->sp; + dapl_log(DAPL_DBG_TYPE_CM, - " dapli_cm_free: cm %p %s ep %p refs=%d\n", + " dapli_cm_free: cm %p %s ep %p sp %p cr_cnt %d refs=%d\n", cm, dapl_cm_state_str(cm->state), - cm->ep, cm->ref_count); + cm->ep, sp_ptr, sp_ptr ? sp_ptr->cr_list_count:0, cm->ref_count); + + if (sp_ptr && cm->state == DCM_CONNECTED && + cm->msg.daddr.ib.qp_type == IBV_QPT_UD) { + DAPL_CR *cr_ptr = dapl_sp_search_cr(sp_ptr, cm); + dapl_log(DAPL_DBG_TYPE_CM, " dapli_cm_free: UD CR %p\n", cr_ptr); + if (cr_ptr != NULL) { + dapl_os_lock(&sp_ptr->header.lock); + dapl_sp_remove_cr(sp_ptr, cr_ptr); + dapl_os_unlock(&sp_ptr->header.lock); + dapls_cr_free(cr_ptr); + } + } dapl_os_lock(&cm->lock); cm->state = DCM_FREE; -- 2.46.0