ucm: add support for IB collective providers

Add collective member address and threading information
on a per transport basis. Call create/free service with
HCA open/close.

Signed-off-by: Arlin Davis <arlin.r.davis@intel.com>

diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h
index efeec4df4e4296b70abc4823ad18c7e78fffb7a3..b5bd082ed39833d72b46158cecba22e76b306a2c 100644 (file)
--- a/dapl/openib_ucm/dapl_ib_util.h
+++ b/dapl/openib_ucm/dapl_ib_util.h
@@ -120,6 +120,23 @@ typedef struct _ib_hca_transport
        uint8_t                 sl;
        uint16_t                pkey;
        int                     pkey_idx;
+#ifdef DAT_IB_COLLECTIVES
+       /* Collective member device and address information */
+       ib_thread_state_t       coll_thread_state;
+       DAPL_OS_THREAD          coll_thread;
+       DAPL_OS_LOCK            coll_lock;
+       DAPL_OS_WAIT_OBJECT     coll_event;
+       struct dapl_llist_entry *grp_list;
+       user_progress_func_t    *user_func;
+       int                     l_sock;
+       struct sockaddr_in      m_addr;
+       void                    *m_ctx;
+       void                    *m_info;
+       void                    *f_info;
+       int                     m_size;
+       int                     f_size;
+       int                     t_id;
+#endif
 
 } ib_hca_transport_t;
 

diff --git a/dapl/openib_ucm/device.c b/dapl/openib_ucm/device.c
index a628a78b87ff1b415c4035db14e39dc101f0517f..a07d886a498e990200af2d10051b62b0fcdeed81 100644 (file)
--- a/dapl/openib_ucm/device.c
+++ b/dapl/openib_ucm/device.c
@@ -33,6 +33,10 @@
 
 #include <stdlib.h>
 
+#ifdef DAT_IB_COLLECTIVES
+#include <collectives/ib_collectives.h>
+#endif
+
 static void ucm_service_destroy(IN DAPL_HCA *hca);
 static int  ucm_service_create(IN DAPL_HCA *hca);
 
@@ -347,6 +351,11 @@ found:
               &hca_ptr->ib_trans.addr, 
               sizeof(union dcm_addr));
 
+#ifdef DAT_IB_COLLECTIVES
+       if (dapli_create_collective_service(hca_ptr))
+               goto bail;
+#endif
+
        ibv_free_device_list(dev_list);
 
        /* wait for cm_thread */
@@ -385,6 +394,10 @@ DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr)
 {
        dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr);
 
+#ifdef DAT_IB_COLLECTIVES
+       dapli_free_collective_service(hca_ptr);
+#endif
+
        if (hca_ptr->ib_trans.cm_state == IB_THREAD_RUN) {
                hca_ptr->ib_trans.cm_state = IB_THREAD_CANCEL;
                dapls_thread_signal(&hca_ptr->ib_trans.signal);