dat/dapl: add support for logging errors in non-debug build.

Add debug logging (stdout, syslog) for error cases during
device open, cm, async, and dto operations. Default settings
are ERR for DAPL_DBG_TYPE, and stdout for DAPL_DBG_DEST.

Add logging by default if library load fails.

Signed-off by: Arlin Davis ardavis@ichips.intel.com

diff --git a/dapl/common/dapl_debug.c b/dapl/common/dapl_debug.c
index 9555f87d7393d7e23a438d1b619813193456e2ea..028f602fcc441a1a41fc1d23f3b5b2d19c7fae2d 100644 (file)
--- a/dapl/common/dapl_debug.c
+++ b/dapl/common/dapl_debug.c
@@ -32,7 +32,6 @@
 #include <stdlib.h>
 #endif /* __KDAPL__ */
 
-#ifdef DAPL_DBG
 DAPL_DBG_TYPE g_dapl_dbg_type;         /* initialized in dapl_init.c */
 DAPL_DBG_DEST g_dapl_dbg_dest;         /* initialized in dapl_init.c */
 
@@ -116,5 +115,4 @@ void dapl_dump_cntr( int cntr )
 }
 
 #endif /* DAPL_COUNTERS */
-#endif
 

diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c
index 346600bf16df8f7dd99aa80d4b6656ad9d8c4498..39a8dd9ff5ab9c1e463ac6aa816d9cfe080b29eb 100644 (file)
--- a/dapl/common/dapl_evd_util.c
+++ b/dapl/common/dapl_evd_util.c
@@ -1165,10 +1165,10 @@ dapli_evd_cqe_to_event (
            dapl_os_unlock ( &ep_ptr->header.lock );
        }
 
-       dapl_dbg_log (DAPL_DBG_TYPE_DTO_COMP_ERR,
-                     " DTO completion ERROR: %d: op %#x (ep disconnected)\n",
-                     DAPL_GET_CQE_STATUS (cqe_ptr),
-                     DAPL_GET_CQE_OPTYPE (cqe_ptr));
+       dapl_log(DAPL_DBG_TYPE_ERR,
+                "DTO completion ERR: status %d, opcode %s \n",
+                DAPL_GET_CQE_STATUS(cqe_ptr),
+                DAPL_GET_CQE_OP_STR(cqe_ptr));
     }
 }
 

diff --git a/dapl/include/dapl_debug.h b/dapl/include/dapl_debug.h
index a3bdcf692c3a3be7bb558cde7f287e59345d2cd9..89c16b46ec64350eaa438a3f9dc8642e1e2095dc 100644 (file)
--- a/dapl/include/dapl_debug.h
+++ b/dapl/include/dapl_debug.h
@@ -75,17 +75,21 @@ typedef enum
     DAPL_DBG_DEST_SYSLOG       = 0x0002,
 } DAPL_DBG_DEST;
 
+extern DAPL_DBG_TYPE    g_dapl_dbg_type;
+extern DAPL_DBG_DEST    g_dapl_dbg_dest;
 
-#if defined(DAPL_DBG)
+extern void dapl_internal_dbg_log(DAPL_DBG_TYPE type,  const char *fmt,  ...);
+
+#define dapl_log g_dapl_dbg_type==0 ? (void) 1 : dapl_internal_dbg_log
 
-extern DAPL_DBG_TYPE   g_dapl_dbg_type;
-extern DAPL_DBG_DEST   g_dapl_dbg_dest;
+#if defined(DAPL_DBG)
 
 #define dapl_dbg_log g_dapl_dbg_type==0 ? (void) 1 : dapl_internal_dbg_log
-extern void dapl_internal_dbg_log ( DAPL_DBG_TYPE type,  const char *fmt,  ...);
 
 #else  /* !DAPL_DBG */
+
 #define dapl_dbg_log(...)
+
 #endif /* !DAPL_DBG */
 
 /*

diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c
index 442efaac9fe5bd06e9fcd19048597bab16c3e42e..f08ee4b591e42af35c9c0a05f4f76580b570932d 100755 (executable)
--- a/dapl/openib_cma/dapl_ib_cm.c
+++ b/dapl/openib_cma/dapl_ib_cm.c
@@ -83,10 +83,12 @@ static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
 /* cma requires 16 bit SID */
 #define IB_PORT_MOD 32001
 #define IB_PORT_BASE (65535 - IB_PORT_MOD)
-#define MAKE_PORT(SID) \
+#define SID_TO_PORT(SID) \
     (SID > 0xffff ? \
-    (unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE) :\
-    (unsigned short)SID)
+    htons((unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE)) :\
+    htons((unsigned short)SID))
+
+#define PORT_TO_SID(p) ntohs(p)
 
 static void dapli_addr_resolve(struct dapl_cm_id *conn)
 {
@@ -104,8 +106,9 @@ static void dapli_addr_resolve(struct dapl_cm_id *conn)
        
        ret =  rdma_resolve_route(conn->cm_id, conn->route_timeout);
        if (ret) {
-               dapl_dbg_log(DAPL_DBG_TYPE_ERR, 
-                            " rdma_connect failed: %s\n",strerror(errno));
+               dapl_log(DAPL_DBG_TYPE_ERR,
+                        " dapl_cma_connect: rdma_resolve_route ERR %d %s\n",
+                        ret, strerror(errno));
 
                dapl_evd_connection_callback(conn, 
                                             IB_CME_LOCAL_FAILURE, 
@@ -155,8 +158,9 @@ static void dapli_route_resolve(struct dapl_cm_id *conn)
        ret = rdma_connect(conn->cm_id, &conn->params);
 
        if (ret) {
-               dapl_dbg_log(DAPL_DBG_TYPE_ERR, " rdma_connect failed: %s\n",
-                            strerror(errno));
+               dapl_log(DAPL_DBG_TYPE_ERR,
+                        " dapl_cma_connect: rdma_connect ERR %d %s\n",
+                        ret, strerror(errno));
                goto bail;
        }
        return;
@@ -319,12 +323,15 @@ static void dapli_cm_active_cb(struct dapl_cm_id *conn,
        case RDMA_CM_EVENT_UNREACHABLE:
        case RDMA_CM_EVENT_CONNECT_ERROR:
        {
-               dapl_dbg_log(
-                        DAPL_DBG_TYPE_WARN,
-                        " dapli_cm_active_handler: CONN_ERR "
-                        " event=0x%x status=%d %s\n",
-                        event->event, event->status,
-                        (event->status == -ETIMEDOUT)?"TIMEOUT":"" );
+               dapl_log(DAPL_DBG_TYPE_WARN,
+                        "dapl_cma_active: CONN_ERR event=0x%x"
+                        " status=%d %s DST %s, %d\n",
+                        event->event, event->status,
+                        (event->status == -ETIMEDOUT)?"TIMEOUT":"",
+                        inet_ntoa(((struct sockaddr_in *)
+                            &conn->cm_id->route.addr.dst_addr)->sin_addr),
+                        ntohs(((struct sockaddr_in *)
+                            &conn->cm_id->route.addr.dst_addr)->sin_port));
 
                /* per DAT SPEC provider always returns UNREACHABLE */
                dapl_evd_connection_callback(conn, 
@@ -339,40 +346,35 @@ static void dapli_cm_active_cb(struct dapl_cm_id *conn,
                /* no device type specified so assume IB for now */
                if (event->status == 28) /* IB_CM_REJ_CONSUMER_DEFINED */
                        cm_event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA;
-               else 
+               else {
                        cm_event = IB_CME_DESTINATION_REJECT;
 
-                dapl_dbg_log(DAPL_DBG_TYPE_CM,
-                       " dapli_cm_active: REJECTED reason=%d"
-                       " SRC 0x%x,%d DST 0x%x,%d\n",
-                       event->status,
-                       ntohl(((struct sockaddr_in *)
-                        &conn->cm_id->route.addr.src_addr)->sin_addr.s_addr),
-                       ntohs(((struct sockaddr_in *)
-                        &conn->cm_id->route.addr.src_addr)->sin_port),
-                       ntohl(((struct sockaddr_in *)
-                        &conn->cm_id->route.addr.dst_addr)->sin_addr.s_addr),
-                       ntohs(((struct sockaddr_in *)
-                        &conn->cm_id->route.addr.dst_addr)->sin_port));
-
+                       dapl_log(DAPL_DBG_TYPE_WARN,
+                           "dapl_cma_active: non-consumer REJ,"
+                           " reason=%d, DST %s, %d\n",
+                           event->status,
+                           inet_ntoa(((struct sockaddr_in *)
+                             &conn->cm_id->route.addr.dst_addr)->sin_addr),
+                           ntohs(((struct sockaddr_in *)
+                             &conn->cm_id->route.addr.dst_addr)->sin_port));
+               }
                dapl_evd_connection_callback(conn, cm_event, NULL, conn->ep);
                break;
        }
        case RDMA_CM_EVENT_ESTABLISHED:
-                       
-               dapl_dbg_log(DAPL_DBG_TYPE_CM, 
-                    " active_cb: cm_id %d PORT %d CONNECTED to 0x%x!\n",
-                    conn->cm_id,
-                    ntohs(((struct sockaddr_in *)
-                       &conn->cm_id->route.addr.dst_addr)->sin_port),
-                    ntohl(((struct sockaddr_in *)
-                       &conn->cm_id->route.addr.dst_addr)->sin_addr.s_addr));
+                dapl_dbg_log(DAPL_DBG_TYPE_CM,
+                     " active_cb: cm_id %d PORT %d CONNECTED to %s!\n",
+                     conn->cm_id,
+                     ntohs(((struct sockaddr_in *)
+                        &conn->cm_id->route.addr.dst_addr)->sin_port),
+                     inet_ntoa(((struct sockaddr_in *)
+                            &conn->cm_id->route.addr.dst_addr)->sin_addr));
 
                /* setup local and remote ports for ep query */
                conn->ep->param.remote_port_qual = 
-                       ntohs(rdma_get_dst_port(conn->cm_id));
+                       PORT_TO_SID(rdma_get_dst_port(conn->cm_id));
                conn->ep->param.local_port_qual = 
-                       ntohs(rdma_get_src_port(conn->cm_id));
+                       PORT_TO_SID(rdma_get_src_port(conn->cm_id));
 
                dapl_evd_connection_callback(conn, IB_CME_CONNECTED,
                                             event->param.conn.private_data, conn->ep);
@@ -429,22 +431,15 @@ static void dapli_cm_passive_cb(struct dapl_cm_id *conn,
                break;
        case RDMA_CM_EVENT_UNREACHABLE:
        case RDMA_CM_EVENT_CONNECT_ERROR:
-
-               dapl_dbg_log(
-                        DAPL_DBG_TYPE_WARN,
-                        " dapli_cm_passive: CONN_ERR "
-                        " event=0x%x status=%d %s"
-                        " on SRC 0x%x,%d DST 0x%x,%d\n",
+                dapl_log(DAPL_DBG_TYPE_WARN,
+                        "dapl_cm_passive: CONN_ERR event=0x%x status=%d %s,"
+                        " DST %s,%d\n",
                         event->event, event->status,
-                        (event->status == -110)?"TIMEOUT":"",
-                        ntohl(((struct sockaddr_in *)
-                                &ipaddr->src_addr)->sin_addr.s_addr),
+                        (event->status == -ETIMEDOUT)?"TIMEOUT":"",
+                        inet_ntoa(((struct sockaddr_in *)
+                            &conn->cm_id->route.addr.dst_addr)->sin_addr),
                         ntohs(((struct sockaddr_in *)
-                                &ipaddr->src_addr)->sin_port),
-                        ntohl(((struct sockaddr_in *)
-                                &ipaddr->dst_addr)->sin_addr.s_addr),
-                        ntohs(((struct sockaddr_in *)
-                                &ipaddr->dst_addr)->sin_port));
+                            &conn->cm_id->route.addr.dst_addr)->sin_port));
 
                dapls_cr_callback(conn, IB_CME_DESTINATION_UNREACHABLE,
                                 NULL, conn->sp);
@@ -452,41 +447,28 @@ static void dapli_cm_passive_cb(struct dapl_cm_id *conn,
 
        case RDMA_CM_EVENT_REJECTED:
        {
-               ib_cm_events_t cm_event;
-
-               /* no device type specified so assume IB for now */
-               if (event->status == 28) /* IB_CM_REJ_CONSUMER_DEFINED */
-                       cm_event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA;
-               else 
-                       cm_event = IB_CME_DESTINATION_REJECT;
+                /* will alwasys be abnormal NON-consumer from active side */
+                dapl_log(DAPL_DBG_TYPE_WARN,
+                        "dapl_cm_passive: non-consumer REJ, reason=%d,"
+                        " DST %s, %d\n",
+                        event->status,
+                        inet_ntoa(((struct sockaddr_in *)
+                        &conn->cm_id->route.addr.dst_addr)->sin_addr),
+                        ntohs(((struct sockaddr_in *)
+                        &conn->cm_id->route.addr.dst_addr)->sin_port));
 
-               dapl_dbg_log(
-                       DAPL_DBG_TYPE_CM, 
-                       " dapli_cm_passive: REJECTED reason=%d"
-                       " on SRC 0x%x,0x%x DST 0x%x,0x%x\n", 
-                       event->status,
-                       ntohl(((struct sockaddr_in *)
-                               &ipaddr->src_addr)->sin_addr.s_addr),
-                       ntohs(((struct sockaddr_in *)
-                               &ipaddr->src_addr)->sin_port),
-                       ntohl(((struct sockaddr_in *)
-                               &ipaddr->dst_addr)->sin_addr.s_addr),
-                       ntohs(((struct sockaddr_in *)
-                               &ipaddr->dst_addr)->sin_port));
-               
-               dapls_cr_callback(conn, cm_event, NULL, conn->sp);
-               
+               dapls_cr_callback(conn, IB_CME_DESTINATION_REJECT, 
+                                 NULL, conn->sp);
                break;
        }
        case RDMA_CM_EVENT_ESTABLISHED:
-               
                dapl_dbg_log(DAPL_DBG_TYPE_CM, 
-                    " passive_cb: cm_id %p PORT %d CONNECTED from 0x%x!\n",
+                    " passive_cb: cm_id %p PORT %d CONNECTED from %s!\n",
                     conn->cm_id,
                     ntohs(((struct sockaddr_in *)
                        &conn->cm_id->route.addr.src_addr)->sin_port),
-                    ntohl(((struct sockaddr_in *)
-                       &conn->cm_id->route.addr.dst_addr)->sin_addr.s_addr));
+                    inet_ntoa(((struct sockaddr_in *)
+                        &conn->cm_id->route.addr.dst_addr)->sin_addr));
 
                dapls_cr_callback(conn, IB_CME_CONNECTED, 
                                  NULL, conn->sp);
@@ -550,7 +532,7 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
                return DAT_SUCCESS;
 
        dapl_dbg_log(DAPL_DBG_TYPE_CM, " connect: rSID %d, pdata %p, ln %d\n", 
-                    r_qual,p_data,p_size);
+                    SID_TO_PORT(r_qual),p_data,p_size);
                        
        /* rdma conn and cm_id pre-bound; reference via qp_handle */
        conn = ep_ptr->cm_handle = ep_ptr->qp_handle;
@@ -572,19 +554,23 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
        dapl_os_memcpy(&conn->r_addr, r_addr, sizeof(*r_addr));
 
        /* Resolve remote address, src already bound during QP create */
-       ((struct sockaddr_in*)&conn->r_addr)->sin_port = htons(MAKE_PORT(r_qual));
+       ((struct sockaddr_in*)&conn->r_addr)->sin_port = SID_TO_PORT(r_qual);
        ((struct sockaddr_in*)&conn->r_addr)->sin_family = AF_INET;
 
        if (rdma_resolve_addr(conn->cm_id, NULL, 
                              (struct sockaddr *)&conn->r_addr, 
-                             conn->arp_timeout))
+                             conn->arp_timeout)) {
+                dapl_log(DAPL_DBG_TYPE_ERR,
+                         " dapl_cma_connect: rdma_resolve_addr ERR %s\n",
+                         strerror(errno));
                return dapl_convert_errno(errno,"ib_connect");
+       }
 
        dapl_dbg_log(DAPL_DBG_TYPE_CM, 
                " connect: resolve_addr: cm_id %p -> %s port %d\n", 
                conn->cm_id, 
                inet_ntoa(((struct sockaddr_in *)&conn->r_addr)->sin_addr),
-               ((struct sockaddr_in*)&conn->r_addr)->sin_port );
+               ntohs(((struct sockaddr_in*)&conn->r_addr)->sin_port));
 
        return DAT_SUCCESS;
 }
@@ -705,7 +691,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr,
        /* open identifies the local device; per DAT specification */
        /* Get family and address then set port to consumer's ServiceID */
        dapl_os_memcpy(&addr, &ia_ptr->hca_ptr->hca_address, sizeof(addr));
-       ((struct sockaddr_in *)&addr)->sin_port = htons(MAKE_PORT(ServiceID));
+       ((struct sockaddr_in *)&addr)->sin_port = SID_TO_PORT(ServiceID);
 
 
        if (rdma_bind_addr(conn->cm_id,(struct sockaddr *)&addr)) {
@@ -719,7 +705,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr,
 
        dapl_dbg_log(DAPL_DBG_TYPE_CM,
                " listen(ia_ptr %p SID %d sp %p conn %p id %d)\n",
-               ia_ptr, MAKE_PORT(ServiceID), 
+               ia_ptr, SID_TO_PORT(ServiceID), 
                sp_ptr, conn, conn->cm_id);
 
        sp_ptr->cm_srvc_handle = conn;
@@ -834,9 +820,9 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
                 */
                dat_status = dapls_ib_qp_alloc(ia_ptr, ep_ptr, NULL);
                if (dat_status != DAT_SUCCESS) {
-                       dapl_dbg_log(DAPL_DBG_TYPE_ERR,
-                                    " accept: ib_qp_alloc failed: %d\n",
-                                    dat_status);
+                       dapl_log(DAPL_DBG_TYPE_ERR,
+                                "dapl_cm_accept: ib_qp_alloc failed: 0x%x\n",
+                                dat_status);
                        goto bail;
                }
        }
@@ -854,11 +840,12 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
                ep_ptr->qp_handle->cm_id->qp = NULL;
                dapli_destroy_conn(ep_ptr->qp_handle);
        } else {
-               dapl_dbg_log(DAPL_DBG_TYPE_ERR, 
-                       " accept: ERR dev(%p!=%p) or port mismatch(%d!=%d)\n", 
-                       ep_ptr->qp_handle->cm_id->verbs,cr_conn->cm_id->verbs,
-                       ep_ptr->qp_handle->cm_id->port_num,
-                       cr_conn->cm_id->port_num );
+                dapl_log(DAPL_DBG_TYPE_ERR,
+                        " dapl_cma_accept: ERR dev(%p!=%p) or"
+                        " port mismatch(%d!=%d)\n",
+                        ep_ptr->qp_handle->cm_id->verbs,cr_conn->cm_id->verbs,
+                        ntohs(ep_ptr->qp_handle->cm_id->port_num),
+                        ntohs(cr_conn->cm_id->port_num));
                dat_status = DAT_INTERNAL_ERROR;
                goto bail;
        }
@@ -869,7 +856,8 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
 
        ret = rdma_accept(cr_conn->cm_id, &cr_conn->params);
        if (ret) {
-               dapl_dbg_log(DAPL_DBG_TYPE_ERR," accept: ERROR %d\n", ret);
+                dapl_log(DAPL_DBG_TYPE_ERR," dapl_cma_accept: ERR %d %s\n",
+                         ret, strerror(errno));
                dat_status = dapl_convert_errno(ret, "accept");
                goto bail;
        }
@@ -880,8 +868,10 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
         cr_conn->ep = ep_ptr;
 
        /* setup local and remote ports for ep query */
-       ep_ptr->param.remote_port_qual = rdma_get_dst_port(cr_conn->cm_id);
-       ep_ptr->param.local_port_qual = rdma_get_src_port(cr_conn->cm_id);
+       ep_ptr->param.remote_port_qual = 
+               PORT_TO_SID(rdma_get_dst_port(cr_conn->cm_id));
+       ep_ptr->param.local_port_qual = 
+               PORT_TO_SID(rdma_get_src_port(cr_conn->cm_id));
 
        return DAT_SUCCESS;
 bail:
@@ -1171,12 +1161,13 @@ void dapli_cma_event_cb(void)
                        break;
 
                case RDMA_CM_EVENT_ADDR_ERROR:
-                       dapl_dbg_log(DAPL_DBG_TYPE_WARN,
-                                    " CM ADDR ERROR: -> %s retry (%d)..\n", 
-                                    inet_ntoa(((struct sockaddr_in *)
-                                       &conn->r_addr)->sin_addr),
-                                       conn->arp_retries);
-                       
+                        dapl_log(DAPL_DBG_TYPE_WARN,
+                                 "dapl_cma_active: CM ADDR ERROR: ->"
+                                 " DST %s retry (%d)..\n",
+                                 inet_ntoa(((struct sockaddr_in *)
+                                        &conn->r_addr)->sin_addr),
+                                 conn->arp_retries);
+
                        /* retry address resolution */
                        if ((--conn->arp_retries) && 
                                (event->status == -ETIMEDOUT)) {
@@ -1196,6 +1187,15 @@ void dapli_cma_event_cb(void)
                                }
                        } 
                        /* retries exhausted or resolve_addr failed */
+                        dapl_log(DAPL_DBG_TYPE_ERR,
+                                "dapl_cma_active: ARP_ERR, retries(%d)"
+                                " exhausted -> DST %s,%d\n",
+                                IB_ARP_RETRY_COUNT,
+                                inet_ntoa(((struct sockaddr_in *)
+                                &conn->cm_id->route.addr.dst_addr)->sin_addr),
+                                ntohs(((struct sockaddr_in *)
+                                &conn->cm_id->route.addr.dst_addr)->sin_port));
+
                        dapl_evd_connection_callback(
                                conn, IB_CME_DESTINATION_UNREACHABLE, 
                                NULL, conn->ep);
@@ -1203,20 +1203,31 @@ void dapli_cma_event_cb(void)
 
 
                case RDMA_CM_EVENT_ROUTE_ERROR:
-                       dapl_dbg_log(DAPL_DBG_TYPE_WARN, 
-                                    " CM ROUTE ERROR: -> %s retry (%d)..\n", 
-                                    inet_ntoa(((struct sockaddr_in *)
-                                       &conn->r_addr)->sin_addr),
-                                    conn->route_retries );
+                        dapl_log(DAPL_DBG_TYPE_WARN,
+                                 "dapl_cma_active: CM ROUTE ERROR: ->"
+                                 " DST %s retry (%d)..\n",
+                                 inet_ntoa(((struct sockaddr_in *)
+                                        &conn->r_addr)->sin_addr),
+                                 conn->route_retries );
 
                        /* retry route resolution */
                        if ((--conn->route_retries) && 
                                (event->status == -ETIMEDOUT))
                                dapli_addr_resolve(conn);
-                       else 
+                       else {
+                            dapl_log(DAPL_DBG_TYPE_ERR,
+                               "dapl_cma_active: PATH_RECORD_ERR,"
+                               " retries(%d) exhausted, DST %s,%d\n",
+                               IB_ROUTE_RETRY_COUNT,
+                               inet_ntoa(((struct sockaddr_in *)
+                               &conn->cm_id->route.addr.dst_addr)->sin_addr),
+                               ntohs(((struct sockaddr_in *)
+                               &conn->cm_id->route.addr.dst_addr)->sin_port));
+
                                dapl_evd_connection_callback( conn, 
                                        IB_CME_DESTINATION_UNREACHABLE, 
                                        NULL, conn->ep);
+                       }
                        break;
                
                case RDMA_CM_EVENT_DEVICE_REMOVAL:

diff --git a/dapl/openib_cma/dapl_ib_dto.h b/dapl/openib_cma/dapl_ib_dto.h
index 2bdd5e0cc46badd70d323d75c34a3b63b708b9cd..1a837182780062ba104af85167d0eb05b6e0dc1d 100644 (file)
--- a/dapl/openib_cma/dapl_ib_dto.h
+++ b/dapl/openib_cma/dapl_ib_dto.h
@@ -275,4 +275,31 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
 #define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t*)cqe_p)->byte_len
 #define DAPL_GET_CQE_IMMED_DATA(cqe_p) ((ib_work_completion_t*)cqe_p)->imm_data
 
+STATIC _INLINE_ char * dapls_dto_op_str(int op)
+{
+    static char *optable[] =
+    {
+        "OP_RDMA_WRITE",
+        "OP_RDMA_WRITE_IMM",
+        "OP_SEND",
+        "OP_SEND_IMM",
+        "OP_RDMA_READ",
+        "OP_COMP_AND_SWAP",
+        "OP_FETCH_AND_ADD",
+        "OP_RECEIVE",
+        "OP_RECEIVE_IMM",
+        "OP_BIND_MW",
+        0
+    };
+    return ((op < 0 || op > 9) ? "Invalid CQE OP?" : optable[op]);
+}
+
+static _INLINE_ char *
+dapls_cqe_op_str(IN ib_work_completion_t *cqe_ptr)
+{
+    return dapls_dto_op_str(DAPL_GET_CQE_OPTYPE(cqe_ptr));
+}
+
+#define DAPL_GET_CQE_OP_STR(cqe) dapls_cqe_op_str(cqe)
+
 #endif /*  _DAPL_IB_DTO_H_ */

diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c
index a262854517f07f2d473d9975a00e1f513a9ddc13..82902c64aa8b6a4165c1c1de5585fb35b4a9418d 100755 (executable)
--- a/dapl/openib_cma/dapl_ib_util.c
+++ b/dapl/openib_cma/dapl_ib_util.c
@@ -123,9 +123,10 @@ static int getipaddr(char *name, char *addr, int len)
                /* retry using network device name */
                ret = getipaddr_netdev(name,addr,len);
                if (ret) {
-                       dapl_dbg_log(DAPL_DBG_TYPE_WARN, 
-                            " getipaddr: invalid name, addr, or netdev(%s)\n",
-                            name);
+                       dapl_log(DAPL_DBG_TYPE_ERR,
+                                " open_hca: getaddr_netdev ERROR:"
+                                " %s. Is %s configured?\n",
+                                strerror(errno), name);
                        return ret;
                }
        } else {
@@ -239,16 +240,21 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA *hca_ptr)
                return DAT_INVALID_ADDRESS;
 
        /* cm_id will bind local device/GID based on IP address */
-       if (rdma_create_id(g_cm_events, &cm_id, (void*)hca_ptr, RDMA_PS_TCP))
+       if (rdma_create_id(g_cm_events, &cm_id, (void*)hca_ptr, RDMA_PS_TCP)) {
+               dapl_log(DAPL_DBG_TYPE_ERR,
+                        " open_hca: rdma_create_id ERR %s\n",
+                        strerror(errno));
                return DAT_INTERNAL_ERROR;
+       }
 
        ret = rdma_bind_addr(cm_id,
                             (struct sockaddr *)&hca_ptr->hca_address);
        if ((ret) || (cm_id->verbs == NULL)) {
                 rdma_destroy_id(cm_id); 
-               dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
-                            " open_hca: ERR bind (%d) %s \n", 
-                            ret, strerror(-ret));
+               dapl_log(DAPL_DBG_TYPE_ERR,
+                        " open_hca: rdma_bind ERR %s."
+                        " Is %s configured?\n",
+                        strerror(errno),hca_name);
                return DAT_INVALID_ADDRESS;
        }
 
@@ -280,11 +286,24 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA *hca_ptr)
        /* EVD events without direct CQ channels, non-blocking */
        hca_ptr->ib_trans.ib_cq = 
                ibv_create_comp_channel(hca_ptr->ib_hca_handle);
+       if (hca_ptr->ib_trans.ib_cq == NULL) {
+               dapl_log(DAPL_DBG_TYPE_ERR,
+                        " open_hca: ibv_create_comp_channel ERR %s\n",
+                        strerror(errno));
+               goto bail;
+       }
+
+       dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+                    " open_hca: CQ channel created(fd=%d)\n",
+                    hca_ptr->ib_trans.ib_cq->fd);
+
        opts = fcntl(hca_ptr->ib_trans.ib_cq->fd, F_GETFL); /* uCQ */
        if (opts < 0 || fcntl(hca_ptr->ib_trans.ib_cq->fd, 
                              F_SETFL, opts | O_NONBLOCK) < 0) {
-               dapl_dbg_log (DAPL_DBG_TYPE_ERR, 
-                             " open_hca: ERR with CQ FD\n" );
+               dapl_log(DAPL_DBG_TYPE_ERR,
+                        " open_hca: fcntl on ib_cq->fd %d ERR %d %s\n",
+                        hca_ptr->ib_trans.ib_cq->fd, opts,
+                       strerror(errno));
                goto bail;
        }
        
@@ -441,20 +460,14 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA *hca_ptr,
                ia_attr->ia_address_ptr = 
                        (DAT_IA_ADDRESS_PTR)&hca_ptr->hca_address;
 
-               dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
-                       " query_hca: %s %s  %d.%d.%d.%d\n", hca_ptr->name,
-                       ((struct sockaddr_in *)
-                       ia_attr->ia_address_ptr)->sin_family == AF_INET ? 
-                       "AF_INET":"AF_INET6",
-                       ((struct sockaddr_in *)
-                       ia_attr->ia_address_ptr)->sin_addr.s_addr >> 0 & 0xff,
-                       ((struct sockaddr_in *)
-                       ia_attr->ia_address_ptr)->sin_addr.s_addr >> 8 & 0xff,
-                       ((struct sockaddr_in *)
-                       ia_attr->ia_address_ptr)->sin_addr.s_addr >> 16 & 0xff,
-                       ((struct sockaddr_in *)
-                       ia_attr->ia_address_ptr)->sin_addr.s_addr >> 24 & 0xff);
-               
+               dapl_log(DAPL_DBG_TYPE_UTIL,
+                        "dapl_query_hca: %s %s %s\n", hca_ptr->name,
+                        ((struct sockaddr_in *)
+                        ia_attr->ia_address_ptr)->sin_family == AF_INET ?
+                        "AF_INET":"AF_INET6",
+                        inet_ntoa(((struct sockaddr_in *)
+                               ia_attr->ia_address_ptr)->sin_addr));
+
                ia_attr->hardware_version_major = dev_attr.hw_ver;
                ia_attr->max_eps                  = dev_attr.max_qp;
                ia_attr->max_dto_per_ep           = dev_attr.max_qp_wr;
@@ -480,22 +493,24 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA *hca_ptr,
                ia_attr->num_vendor_attr          = 0;
                ia_attr->vendor_attr              = NULL;
                ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge;
+               ia_attr->max_iov_segments_per_rdma_write = dev_attr.max_sge;
                /* save rd_atom for peer validation during connect requests */
                hca_ptr->ib_trans.max_rdma_rd_in  = dev_attr.max_qp_rd_atom;
                hca_ptr->ib_trans.max_rdma_rd_out = dev_attr.max_qp_rd_atom;
 
-               dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
-                       " query_hca: (ver=%x) ep %d ep_q %d evd %d evd_q %d\n", 
-                       ia_attr->hardware_version_major,
-                       ia_attr->max_eps, ia_attr->max_dto_per_ep,
-                       ia_attr->max_evds, ia_attr->max_evd_qlen );
-               dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
-                       " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d"
-                       " rd_io %d inline=%d\n",
-                       ia_attr->max_mtu_size, ia_attr->max_rdma_size,
-                       ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs,
-                       ia_attr->max_rmrs, ia_attr->max_rdma_read_per_ep_in,
-                       hca_ptr->ib_trans.max_inline_send);
+               dapl_log(DAPL_DBG_TYPE_UTIL,
+                        "dapl_query_hca: (ver=%x) ep's %d ep_q %d"
+                        " evd's %d evd_q %d\n",
+                        ia_attr->hardware_version_major,
+                        ia_attr->max_eps, ia_attr->max_dto_per_ep,
+                        ia_attr->max_evds, ia_attr->max_evd_qlen );
+               dapl_log(DAPL_DBG_TYPE_UTIL,
+                        "dapl_query_hca: msg %llu rdma %llu iov's %d"
+                        " lmr %d rmr %d rd_io %d inline=%d\n",
+                        ia_attr->max_mtu_size, ia_attr->max_rdma_size,
+                        ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs,
+                        ia_attr->max_rmrs, ia_attr->max_rdma_read_per_ep_in,
+                        hca_ptr->ib_trans.max_inline_send);
        }
        
        if (ep_attr != NULL) {
@@ -510,11 +525,12 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA *hca_ptr,
                ep_attr->max_rdma_read_out= dev_attr.max_qp_rd_atom;
                ep_attr->max_rdma_read_iov= dev_attr.max_sge;
                ep_attr->max_rdma_write_iov= dev_attr.max_sge;
-               dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
-                       " query_hca: MAX msg %llu dto %d iov %d rdma i%d,o%d\n", 
-                       ep_attr->max_mtu_size,
-                       ep_attr->max_recv_dtos, ep_attr->max_recv_iov,
-                       ep_attr->max_rdma_read_in, ep_attr->max_rdma_read_out);
+               dapl_log(DAPL_DBG_TYPE_UTIL,
+                        "dapl_query_hca: MAX msg %llu dto %d iov %d"
+                        " rdma i%d,o%d\n",
+                        ep_attr->max_mtu_size,
+                        ep_attr->max_recv_dtos, ep_attr->max_recv_iov,
+                        ep_attr->max_rdma_read_in, ep_attr->max_rdma_read_out);
        }
        return DAT_SUCCESS;
 }
@@ -692,9 +708,9 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
                        struct dapl_ep *evd_ptr = 
                                event.element.cq->cq_context;
 
-                       dapl_dbg_log(
-                               DAPL_DBG_TYPE_WARN,
-                               " async_event CQ (%p) ERR %d\n",
+                       dapl_log(
+                               DAPL_DBG_TYPE_ERR,
+                               "dapl async_event CQ (%p) ERR %d\n",
                                evd_ptr, event.event_type);                             
                        
                        /* report up if async callback still setup */
@@ -708,9 +724,10 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
                case    IBV_EVENT_COMM_EST:
                {
                        /* Received msgs on connected QP before RTU */
-                       dapl_dbg_log(
+                       dapl_log(
                                DAPL_DBG_TYPE_UTIL,
-                               " async_event COMM_EST(%p) rdata beat RTU\n",
+                               "dapl async_event COMM_EST(%p) "
+                               "rdata beat RTU\n",
                                event.element.qp);      
 
                        break;
@@ -726,9 +743,9 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
                        struct dapl_ep *ep_ptr = 
                                event.element.qp->qp_context;
 
-                       dapl_dbg_log(
-                               DAPL_DBG_TYPE_WARN,
-                               " async_event QP (%p) ERR %d\n",
+                       dapl_log(
+                               DAPL_DBG_TYPE_ERR,
+                               "dapl async_event QP (%p) ERR %d\n",
                                ep_ptr, event.event_type);      
                        
                        /* report up if async callback still setup */
@@ -748,8 +765,8 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
                case    IBV_EVENT_PKEY_CHANGE:
                case    IBV_EVENT_SM_CHANGE:
                {
-                       dapl_dbg_log(DAPL_DBG_TYPE_WARN,
-                                    " async_event: DEV ERR %d\n",
+                       dapl_log(DAPL_DBG_TYPE_WARN,
+                                    "dapl async_event: DEV ERR %d\n",
                                     event.event_type); 
 
                        /* report up if async callback still setup */
@@ -762,13 +779,14 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
                }
                case    IBV_EVENT_CLIENT_REREGISTER:
                        /* no need to report this event this time */
-                       dapl_dbg_log (DAPL_DBG_TYPE_WARN,
-                                    " async_event: IBV_EVENT_CLIENT_REREGISTER\n");
+                       dapl_log (DAPL_DBG_TYPE_UTIL,
+                                 "dapl async_event: "
+                                 "IBV_EVENT_CLIENT_REREGISTER\n");
                        break;
 
                default:
-                       dapl_dbg_log (DAPL_DBG_TYPE_WARN,
-                                    " async_event: %d UNKNOWN\n", 
+                       dapl_log (DAPL_DBG_TYPE_WARN,
+                                    "dapl async_event: %d UNKNOWN\n", 
                                     event.event_type);
                        break;
                

diff --git a/dapl/udapl/dapl_init.c b/dapl/udapl/dapl_init.c
index 787261bdd8d5204d91d4c1986cfca37c6bb8ccf0..20b93cb82440bcb4de06d6f14b227225028d7672 100644 (file)
--- a/dapl/udapl/dapl_init.c
+++ b/dapl/udapl/dapl_init.c
@@ -72,14 +72,19 @@ void dapl_init ( void )
 
     dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: Started (dapl_init)\n");
 
-#if defined(DAPL_DBG)
     /* set up debug type */
     g_dapl_dbg_type = dapl_os_get_env_val ( "DAPL_DBG_TYPE", 
-                                           DAPL_DBG_TYPE_ERR | DAPL_DBG_TYPE_WARN);
+                                           DAPL_DBG_TYPE_ERR );
     /* set up debug destination */
     g_dapl_dbg_dest = dapl_os_get_env_val ( "DAPL_DBG_DEST", 
                                            DAPL_DBG_DEST_STDOUT );
-#endif /* DAPL_DBG */
+
+    /* open log file on first logging call if necessary */
+    if (g_dapl_dbg_dest & DAPL_DBG_DEST_SYSLOG)
+        openlog("libdapl", LOG_ODELAY|LOG_PID|LOG_CONS, LOG_USER);
+
+    dapl_log (DAPL_DBG_TYPE_UTIL, "dapl_init: dbg_type=0x%x,dbg_dest=0x%x\n",
+              g_dapl_dbg_type, g_dapl_dbg_dest);
 
     /* See if the user is on a loopback setup */
     g_dapl_loopback_connection = dapl_os_get_env_bool ( "DAPL_LOOPBACK" );
@@ -142,7 +147,7 @@ void dapl_fini ( void )
 {
     DAT_RETURN         dat_status;
 
-    dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: Stopped (dapl_fini)\n");
+    dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: Enter (dapl_fini)\n");
 
     dat_status = dapl_provider_list_destroy ();
     if (DAT_SUCCESS != dat_status)
@@ -153,6 +158,11 @@ void dapl_fini ( void )
     
     dapls_ib_release ();
 
+    dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "DAPL: Exit (dapl_fini)\n");
+
+    if (g_dapl_dbg_dest & DAPL_DBG_DEST_SYSLOG)
+        closelog();
+
     return;
 }
 

diff --git a/dapl/udapl/linux/dapl_osd.h b/dapl/udapl/linux/dapl_osd.h
index e932ae0d28f3921831c8877596a130b47c350024..580c8ad4bc78646ccbd165e2a6f6e85267cbe82b 100644 (file)
--- a/dapl/udapl/linux/dapl_osd.h
+++ b/dapl/udapl/linux/dapl_osd.h
@@ -537,13 +537,10 @@ dapl_os_strtol(const char *nptr, char **endptr, int base)
  *  Helper Functions
  */
 
-
 #define dapl_os_assert(expression)     assert(expression)
 #define dapl_os_printf(...)            printf(__VA_ARGS__)
 #define dapl_os_vprintf(fmt,args)      vprintf(fmt,args)
-#define dapl_os_syslog(fmt,args)       vsyslog (LOG_USER | LOG_DEBUG,fmt,args)
-
-
+#define dapl_os_syslog(fmt,args)       vsyslog(LOG_USER|LOG_WARNING,fmt,args)
 
 #endif /*  _DAPL_OSD_H_ */
 

diff --git a/dat/udat/linux/dat_osd.c b/dat/udat/linux/dat_osd.c
index bc66828ffd0d3aa16181cca516a7a9b0a46555e3..d6a574728bb22eb6e4bff88ed4fb9e0fe99bbf7e 100644 (file)
--- a/dat/udat/linux/dat_osd.c
+++ b/dat/udat/linux/dat_osd.c
@@ -156,8 +156,8 @@ dat_os_library_load (
     }
     else
     {
-       dat_os_dbg_print (DAT_OS_DBG_TYPE_SR,
-                        "DAT: library load failure: %s\n",
+       dat_os_dbg_print(DAT_OS_DBG_TYPE_ERROR,
+                        "DAT: library load failure: %s\n",
                         dlerror ());
        return DAT_INTERNAL_ERROR;
     }