From: Arlin Davis Date: Fri, 12 May 2006 19:50:19 +0000 (+0000) Subject: r7141: Update the uDAPL openib_cma provider to work with the new X-Git-Tag: libdapl-1.2.1~40 X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=b1b6e16f3e41e123cd347bc78b01e3272076362b;p=~ardavis%2Fdapl.git r7141: Update the uDAPL openib_cma provider to work with the new uCMA event channel interface. Signed-off-by: Arlin Davis Signed-off-by: James Lentini --- diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c index ae90f6e..bdc1034 100644 --- a/dapl/openib_cma/dapl_ib_cm.c +++ b/dapl/openib_cma/dapl_ib_cm.c @@ -59,6 +59,8 @@ #include #include +extern struct rdma_event_channel *g_cm_events; + /* local prototypes */ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn, struct rdma_cm_event *event); @@ -614,7 +616,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr, dapl_os_lock_init(&conn->lock); /* create CM_ID, bind to local device, create QP */ - if (rdma_create_id(&conn->cm_id, (void*)conn)) { + if (rdma_create_id(g_cm_events, &conn->cm_id, (void*)conn)) { dapl_os_free(conn, sizeof(*conn)); return(dapl_convert_errno(errno,"setup_listener")); } @@ -1067,7 +1069,7 @@ void dapli_cma_event_cb(void) dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_event()\n"); /* process one CM event, fairness */ - if(!rdma_get_cm_event(&event)) { + if(!rdma_get_cm_event(g_cm_events, &event)) { struct dapl_cm_id *conn; /* set proper conn from cm_id context*/ diff --git a/dapl/openib_cma/dapl_ib_qp.c b/dapl/openib_cma/dapl_ib_qp.c index 9015fd6..9a91db5 100644 --- a/dapl/openib_cma/dapl_ib_qp.c +++ b/dapl/openib_cma/dapl_ib_qp.c @@ -35,6 +35,8 @@ #include "dapl.h" #include "dapl_adapter_util.h" +extern struct rdma_event_channel *g_cm_events; + /* * dapl_ib_qp_alloc * @@ -128,7 +130,7 @@ DAT_RETURN dapls_ib_qp_alloc(IN DAPL_IA *ia_ptr, dapl_os_lock_init(&conn->lock); /* create CM_ID, bind to local device, create QP */ - if (rdma_create_id(&cm_id, (void*)conn)) { + if (rdma_create_id(g_cm_events, &cm_id, (void*)conn)) { dapl_os_free(conn, sizeof(*conn)); return(dapl_convert_errno(errno, "create_qp")); } diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c index 56e896a..b733ebb 100644 --- a/dapl/openib_cma/dapl_ib_util.c +++ b/dapl/openib_cma/dapl_ib_util.c @@ -67,6 +67,7 @@ static const char rcsid[] = "$Id: $"; int g_dapl_loopback_connection = 0; int g_ib_pipe[2]; +struct rdma_event_channel *g_cm_events = NULL; ib_thread_state_t g_ib_thread_state = 0; DAPL_OS_THREAD g_ib_thread; DAPL_OS_LOCK g_hca_lock; @@ -184,6 +185,7 @@ int32_t dapls_ib_release(void) { dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n"); dapli_ib_thread_destroy(); + rdma_destroy_event_channel(g_cm_events); return 0; } @@ -214,9 +216,17 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA *hca_ptr) dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: %s - %p\n", hca_name, hca_ptr); + /* Setup the global cm event channel */ + dapl_os_lock(&g_hca_lock); + if (g_cm_events == NULL) { + g_cm_events = rdma_create_event_channel(); + if (g_cm_events == NULL) + return DAT_INTERNAL_ERROR; + } + dapl_os_unlock(&g_hca_lock); + if (dapli_ib_thread_init()) return DAT_INTERNAL_ERROR; - /* HCA name will be hostname or IP address */ if (getipaddr((char*)hca_name, @@ -224,14 +234,13 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA *hca_ptr) sizeof(DAT_SOCK_ADDR6))) return DAT_INVALID_ADDRESS; - /* cm_id will bind local device/GID based on IP address */ - if (rdma_create_id(&cm_id, (void*)hca_ptr)) + if (rdma_create_id(g_cm_events, &cm_id, (void*)hca_ptr)) return DAT_INTERNAL_ERROR; ret = rdma_bind_addr(cm_id, (struct sockaddr *)&hca_ptr->hca_address); - if (ret) { + if ((ret) || (cm_id->verbs == NULL)) { rdma_destroy_id(cm_id); dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: ERR bind (%d) %s \n", @@ -551,8 +560,8 @@ int dapli_ib_thread_init(void) } /* uCMA events non-blocking */ - opts = fcntl(rdma_get_fd(), F_GETFL); /* uCMA */ - if (opts < 0 || fcntl(rdma_get_fd(), + opts = fcntl(g_cm_events->fd, F_GETFL); /* uCMA */ + if (opts < 0 || fcntl(g_cm_events->fd, F_SETFL, opts | O_NONBLOCK) < 0) { dapl_dbg_log (DAPL_DBG_TYPE_ERR, " dapl_ib_init: ERR with uCMA FD\n" ); @@ -741,7 +750,7 @@ void dapli_thread(void *arg) dapl_dbg_log (DAPL_DBG_TYPE_UTIL, " ib_thread(%d,0x%x): ENTER: pipe %d ucma %d\n", - getpid(), g_ib_thread, g_ib_pipe[0], rdma_get_fd()); + getpid(), g_ib_thread, g_ib_pipe[0], g_cm_events->fd); /* Poll across pipe, CM, AT never changes */ dapl_os_lock( &g_hca_lock ); @@ -749,7 +758,7 @@ void dapli_thread(void *arg) ufds[0].fd = g_ib_pipe[0]; /* pipe */ ufds[0].events = POLLIN; - ufds[1].fd = rdma_get_fd(); /* uCMA */ + ufds[1].fd = g_cm_events->fd; /* uCMA */ ufds[1].events = POLLIN; while (g_ib_thread_state == IB_THREAD_RUN) {