uDAPL: scm provider, remove query gid/lid from connection setup phase

move lid/gid queries from the connection setup phase
and put them in the open call to avoid overhead
of more fd's during connections. No need
to query during connection setup since uDAPL
binds to specific hca/ports via dat_ia_open.

Signed-off-by: Arlin Davis <ardavis@ichips.intel.com>

diff --git a/dapl/openib_scm/dapl_ib_cm.c b/dapl/openib_scm/dapl_ib_cm.c
index 9a15e42282d82a100200ae416e5083205843cdb3..9defc4240eb60a9562e8e2097baea0fe47e6270d 100644 (file)
--- a/dapl/openib_scm/dapl_ib_cm.c
+++ b/dapl/openib_scm/dapl_ib_cm.c
@@ -285,16 +285,6 @@ static void dapli_cm_queue(struct ib_cm_handle *cm_ptr)
                         strerror(errno));
 }
 
-static uint16_t dapli_get_lid(IN struct ibv_context *ctx, IN uint8_t port)
-{
-       struct ibv_port_attr port_attr;
-
-       if(ibv_query_port(ctx, port,&port_attr))
-               return(0xffff);
-       else
-               return(port_attr.lid);
-}
-
 /*
  * ACTIVE/PASSIVE: called from CR thread or consumer via ep_disconnect
  */
@@ -468,28 +458,9 @@ dapli_socket_connect(DAPL_EP               *ep_ptr,
        cm_ptr->dst.qp_type = htons(ep_ptr->qp_handle->qp_type);
 #endif
        cm_ptr->dst.port = htons(ia_ptr->hca_ptr->port_num);
-       cm_ptr->dst.lid = 
-               htons(dapli_get_lid(ia_ptr->hca_ptr->ib_hca_handle, 
-                                   (uint8_t)ia_ptr->hca_ptr->port_num));
-       if (cm_ptr->dst.lid == 0xffff) {
-               dapl_log(DAPL_DBG_TYPE_ERR, 
-                        " CONNECT: query LID ERR %s -> %s\n",
-                        strerror(errno), 
-                        inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr));
-               goto bail;
-       }
-
-        /* in network order */
-        if (ibv_query_gid(ia_ptr->hca_ptr->ib_hca_handle,
-                                   (uint8_t)ia_ptr->hca_ptr->port_num,
-                                   0, &cm_ptr->dst.gid)) {
-               dapl_log(DAPL_DBG_TYPE_ERR, 
-                        " CONNECT: query GID ERR %s -> %s\n",
-                        strerror(errno), 
-                        inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr));
-               goto bail;
-       }
-
+       cm_ptr->dst.lid = ia_ptr->hca_ptr->ib_trans.lid;
+       cm_ptr->dst.gid = ia_ptr->hca_ptr->ib_trans.gid;
+        
        /* save references */
        cm_ptr->hca = ia_ptr->hca_ptr;
        cm_ptr->ep = ep_ptr;
@@ -958,29 +929,8 @@ dapli_socket_accept_usr(DAPL_EP            *ep_ptr,
        local.qpn = htonl(ep_ptr->qp_handle->qp_num);
        local.qp_type = htons(ep_ptr->qp_handle->qp_type);
        local.port = htons(ia_ptr->hca_ptr->port_num);
-       local.lid = htons(dapli_get_lid(ia_ptr->hca_ptr->ib_hca_handle, 
-                                       (uint8_t)ia_ptr->hca_ptr->port_num));
-       if (local.lid == 0xffff) {
-               dapl_log(DAPL_DBG_TYPE_ERR, 
-                        " ACCEPT_USR: query LID ERR %s -> %s\n",
-                        strerror(errno), 
-                        inet_ntoa(((struct sockaddr_in *)
-                               &cm_ptr->dst.ia_address)->sin_addr)); 
-               goto bail;
-       }
-
-        /* in network order */
-       if (ibv_query_gid(ia_ptr->hca_ptr->ib_hca_handle,
-                         (uint8_t)ia_ptr->hca_ptr->port_num,
-                         0, &local.gid)) {
-               dapl_log(DAPL_DBG_TYPE_ERR, 
-                        " ACCEPT_USR: query GID ERR %s -> %s\n",
-                        strerror(errno), 
-                        inet_ntoa(((struct sockaddr_in *)
-                               &cm_ptr->dst.ia_address)->sin_addr)); 
-               goto bail;
-       }
-
+       local.lid = ia_ptr->hca_ptr->ib_trans.lid; 
+       local.gid = ia_ptr->hca_ptr->ib_trans.gid; 
        local.ia_address = ia_ptr->hca_ptr->hca_address;
        local.p_size = htonl(p_size);
        iov[0].iov_base = (void *) &local;

diff --git a/dapl/openib_scm/dapl_ib_util.c b/dapl/openib_scm/dapl_ib_util.c
index fdbae3cfa29b83d997461b989963e34f915fb76b..391a0e54a88d2bffd46b080c5b555792653b699b 100644 (file)
--- a/dapl/openib_scm/dapl_ib_util.c
+++ b/dapl/openib_scm/dapl_ib_util.c
@@ -196,6 +196,7 @@ DAT_RETURN dapls_ib_open_hca (
         IN   DAPL_HCA          *hca_ptr)
 {
        struct ibv_device **dev_list;
+       struct ibv_port_attr port_attr;
        int             i;
        DAT_RETURN      dat_status = DAT_SUCCESS;
 
@@ -238,6 +239,30 @@ found:
                 goto err;
        }
 
+       /* get lid for this hca-port, network order */
+       if(ibv_query_port(hca_ptr->ib_hca_handle, 
+                         (uint8_t)hca_ptr->port_num, 
+                         &port_attr)) {
+               dapl_log(DAPL_DBG_TYPE_ERR, 
+                         " open_hca: get lid ERR for %s, err=%s\n", 
+                         ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
+                         strerror(errno));
+                goto err;
+       } else {
+               hca_ptr->ib_trans.lid = htons(port_attr.lid);
+       }
+
+       /* get gid for this hca-port, network order */
+        if (ibv_query_gid(hca_ptr->ib_hca_handle,
+                         (uint8_t)hca_ptr->port_num,
+                          0, &hca_ptr->ib_trans.gid)) {
+               dapl_log(DAPL_DBG_TYPE_ERR, 
+                        " open_hca: query GID ERR for %s, err=%s\n",
+                        ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
+                        strerror(errno));
+               goto err;
+       }
+
        /* set RC tunables via enviroment or default */
        hca_ptr->ib_trans.max_inline_send = 
                dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT);
@@ -325,6 +350,13 @@ found:
                     hca_ptr->port_num,
                     inet_ntoa(((struct sockaddr_in *)
                                &hca_ptr->hca_address)->sin_addr));
+       dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
+                    " open_hca: LID 0x%x GID Subnet 0x"F64x" ID 0x"F64x"\n",  
+                    ntohs(hca_ptr->ib_trans.lid), 
+                    (unsigned long long) 
+                       htonll(hca_ptr->ib_trans.gid.global.subnet_prefix),
+                     (unsigned long long) 
+                       htonll(hca_ptr->ib_trans.gid.global.interface_id));
                
        ibv_free_device_list(dev_list);
        return dat_status;

diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h
index fd1c24e971b5ec97dbc4e10103ebecb5d7874c1d..7c3251a0305c11511a2ec0c87352d5895966fdb5 100644 (file)
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -277,12 +277,13 @@ typedef enum
 /* ib_hca_transport_t, specific to this implementation */
 typedef struct _ib_hca_transport
 { 
+       union ibv_gid           gid;
        struct  ibv_device      *ib_dev;
        ib_cq_handle_t          ib_cq_empty;
        DAPL_OS_LOCK            cq_lock;        
        int                     max_inline_send;
        ib_thread_state_t       cq_state;
-       DAPL_OS_THREAD                  cq_thread;
+       DAPL_OS_THREAD          cq_thread;
        struct ibv_comp_channel *ib_cq;
        int                     cr_state;
        DAPL_OS_THREAD          thread;
@@ -293,6 +294,7 @@ typedef struct _ib_hca_transport
        ib_async_cq_handler_t   async_cq_error;
        ib_async_dto_handler_t  async_cq;
        ib_async_qp_handler_t   async_qp_error;
+       uint16_t                lid;
        uint8_t                 ack_timer;
        uint8_t                 ack_retry;
        uint8_t                 rnr_timer;