From f0bf59e9bf295ede582bdb3a74450f7b73de0926 Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Mon, 29 Oct 2012 11:54:46 -0700 Subject: [PATCH] common: 2 new debug logging levels for low system memory and package info DAPL_DBG_TYPE_SYS_WARN = 0x800000 DAPL_DBG_TYPE_VER = 0x1000000 export DAPL_DBG_SYS_MEM = 5 will set the checking for memory less than 5% when DAPL_DBG_TYPE is set with bit DAPL_DBG_TYPE_SYS_WARN. The package must be built with --enable-counters for memory checking to be enabled. In addition, if DAPL_DBG_TYPE -s set with bit DAPL_DBG_TYPE_VER than the package rev and build date will be sent to stdout during library init. Signed-off-by: Arlin Davis --- Makefile.am | 1 + dapl/common/dapl_debug.c | 1 + dapl/include/dapl_debug.h | 3 +++ dapl/openib_common/dapl_ib_dto.h | 17 +++++++++++++++-- dapl/openib_ucm/dapl_ib_util.h | 2 +- dapl/udapl/dapl_init.c | 7 +++++++ dapl/udapl/dapl_lmr_create.c | 4 +++- dapl/udapl/linux/dapl_osd.h | 24 +++++++++++++++++++++++- 8 files changed, 54 insertions(+), 5 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9bd3207..c0f1972 100755 --- a/Makefile.am +++ b/Makefile.am @@ -53,6 +53,7 @@ AM_CFLAGS = -g -Wall -D_GNU_SOURCE -DDAT_CONF="\"$(sysconfdir)/dat.conf\"" endif AM_CFLAGS += -DMPXYD_CONF="\"$(sysconfdir)/mpxyd.conf\"" +AM_CFLAGS += -DPACKAGE_DATE=$$(date +'%Y%m%d') sysconf_DATA = doc/dat.conf doc/mpxyd.conf diff --git a/dapl/common/dapl_debug.c b/dapl/common/dapl_debug.c index cb45496..ce2ae53 100644 --- a/dapl/common/dapl_debug.c +++ b/dapl/common/dapl_debug.c @@ -33,6 +33,7 @@ DAPL_DBG_TYPE g_dapl_dbg_type; /* initialized in dapl_init.c */ DAPL_DBG_DEST g_dapl_dbg_dest; /* initialized in dapl_init.c */ +int g_dapl_dbg_mem; /* initialized in dapl_init.c */ static char *_ptr_host_ = NULL; static char _hostname_[128]; diff --git a/dapl/include/dapl_debug.h b/dapl/include/dapl_debug.h index 6cbe028..15370ea 100644 --- a/dapl/include/dapl_debug.h +++ b/dapl/include/dapl_debug.h @@ -76,6 +76,8 @@ typedef enum DAPL_DBG_TYPE_LINK_ERRS = 0x100000, DAPL_DBG_TYPE_LINK_WARN = 0x200000, DAPL_DBG_TYPE_DIAG_ERRS = 0x400000, + DAPL_DBG_TYPE_SYS_WARN = 0x800000, + DAPL_DBG_TYPE_VER = 0x1000000, } DAPL_DBG_TYPE; @@ -87,6 +89,7 @@ typedef enum extern DAPL_DBG_TYPE g_dapl_dbg_type; extern DAPL_DBG_DEST g_dapl_dbg_dest; +extern int g_dapl_dbg_mem; extern void dapl_internal_dbg_log(DAPL_DBG_TYPE type, const char *fmt, ...); diff --git a/dapl/openib_common/dapl_ib_dto.h b/dapl/openib_common/dapl_ib_dto.h index 296d8f3..ef7ddb9 100644 --- a/dapl/openib_common/dapl_ib_dto.h +++ b/dapl/openib_common/dapl_ib_dto.h @@ -38,6 +38,19 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p); #define CQE_WR_TYPE_UD(id) \ (((DAPL_COOKIE *)(uintptr_t)id)->ep->qp_handle->qp->qp_type == IBV_QPT_UD) +#if defined(_OPENIB_MCM_) +#define PROVIDER_NAME "MCM" +#elif defined(_OPENIB_CMA_) +#define PROVIDER_NAME "CMA" +#elif defined(_OPENIB_UCM_) +#define PROVIDER_NAME "UCM" +#elif defined(_OPENIB_SCM_) +#define PROVIDER_NAME "SCM" +#else +#define PROVIDER_NAME "" +#endif + + /* * dapls_ib_post_recv * @@ -151,12 +164,12 @@ dapls_ib_post_send ( wr.wr.rdma.rkey, wr.wr.rdma.remote_addr); } - +#ifndef _OPENIB_MCM_ /* don't use inline for MCM, optimized for large messages */ /* inline data for send or write ops */ if ((total_len <= ibt_ptr->max_inline_send) && ((op_type == OP_SEND) || (op_type == OP_RDMA_WRITE))) wr.send_flags |= IBV_SEND_INLINE; - +#endif /* set completion flags in work request */ wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & completion_flags) ? 0 : IBV_SEND_SIGNALED; diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h index b5bd082..4aa1ec6 100644 --- a/dapl/openib_ucm/dapl_ib_util.h +++ b/dapl/openib_ucm/dapl_ib_util.h @@ -27,7 +27,7 @@ #ifndef _DAPL_IB_UTIL_H_ #define _DAPL_IB_UTIL_H_ -#define _OPENIB_SCM_ +#define _OPENIB_UCM_ #include #include "openib_osd.h" diff --git a/dapl/udapl/dapl_init.c b/dapl/udapl/dapl_init.c index 07e5b31..b0554f8 100644 --- a/dapl/udapl/dapl_init.c +++ b/dapl/udapl/dapl_init.c @@ -47,6 +47,8 @@ #include "dapl_name_service.h" #include "dapl_timer_util.h" #include "dapl_vendor.h" +#include "config.h" + /* * dapl_init @@ -76,6 +78,8 @@ void dapl_init(void) g_dapl_dbg_dest = dapl_os_get_env_val("DAPL_DBG_DEST", DAPL_DBG_DEST_STDOUT); + g_dapl_dbg_mem = dapl_os_get_env_val("DAPL_DBG_SYS_MEM", 5); + /* open log file on first logging call if necessary */ if (g_dapl_dbg_dest & DAPL_DBG_DEST_SYSLOG) openlog("libdapl", LOG_ODELAY | LOG_PID | LOG_CONS, LOG_USER); @@ -83,6 +87,9 @@ void dapl_init(void) dapl_log(DAPL_DBG_TYPE_UTIL, "dapl_init: dbg_type=0x%x,dbg_dest=0x%x\n", g_dapl_dbg_type, g_dapl_dbg_dest); + dapl_log(DAPL_DBG_TYPE_VER, " %s dapl-%s-%u, DAPL_DBG_TYPE 0x%x\n", + PROVIDER_NAME, VERSION, PACKAGE_DATE, g_dapl_dbg_type); + /* See if the user is on a loopback setup */ g_dapl_loopback_connection = dapl_os_get_env_bool("DAPL_LOOPBACK"); diff --git a/dapl/udapl/dapl_lmr_create.c b/dapl/udapl/dapl_lmr_create.c index f1d5016..86ee672 100644 --- a/dapl/udapl/dapl_lmr_create.c +++ b/dapl/udapl/dapl_lmr_create.c @@ -122,7 +122,9 @@ dapli_lmr_create_virtual(IN DAPL_IA * ia, lmr, virt_addr, length, privileges, va_type); - +#ifdef DAPL_COUNTERS + dapl_os_memchk(g_dapl_dbg_mem, "create_mr"); +#endif if (DAT_SUCCESS != dat_status) { dapl_lmr_dealloc(lmr); goto bail; diff --git a/dapl/udapl/linux/dapl_osd.h b/dapl/udapl/linux/dapl_osd.h index 7198439..34ea2cc 100644 --- a/dapl/udapl/linux/dapl_osd.h +++ b/dapl/udapl/linux/dapl_osd.h @@ -90,6 +90,7 @@ #include #include #include +#include #include #if !defined(SUSE_11) && !defined(REDHAT_EL5) && defined(__ia64__) @@ -434,6 +435,8 @@ dapl_os_wait_object_destroy ( * Memory Functions */ +STATIC _INLINE_ void dapl_os_memchk(int percent, const char *txt); + /* function prototypes */ STATIC _INLINE_ void *dapl_os_alloc (int size); @@ -448,10 +451,29 @@ STATIC _INLINE_ void * dapl_os_memcpy (void *dest, const void *src, int len); STATIC _INLINE_ int dapl_os_memcmp (const void *mem1, const void *mem2, int len); /* memory functions */ - +STATIC _INLINE_ void dapl_os_memchk(int percent, const char *txt) +{ + struct sysinfo si; + double mfree, threshold = (double)percent/100; + + sysinfo(&si); + mfree = si.freeram + si.sharedram + si.bufferram; + + if (mfree/(double)si.totalram < threshold) { + dapl_log(DAPL_DBG_TYPE_SYS_WARN, + " WARNING: LOW MEMORY: %s (Free %d Share %d, Bufs %d)" + " < %2.2f percent of total (%d MB) memory\n", + txt, si.freeram/(1024*1024), si.sharedram/(1024*1024), + si.bufferram/(1024*1024), si.totalram/(1024*1024), + (mfree/(double)si.totalram)*100); + } +} STATIC _INLINE_ void *dapl_os_alloc (int size) { +#ifdef DAPL_COUNTERS + dapl_os_memchk(g_dapl_dbg_mem, "malloc"); +#endif return malloc (size); } -- 2.46.0