dapl: locking cleanup and fixes

Cleanup allocated completion channels. Destroy cm_ptr locks before freeing the cm_ptr to avoid memory leaks.  And avoid accessing the cm_ptr after queuing it for destruction with the cr_thread to avoid use after free errors.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1@2701 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/ulp/dapl2/dapl/openib_cma/cm.c b/trunk/ulp/dapl2/dapl/openib_cma/cm.c
index b8f7198892fe24b2110a76bb081ceb32fd86d1ac..bb35c458c2285676499da6507f96e585e6dfadd4 100644 (file)
--- a/trunk/ulp/dapl2/dapl/openib_cma/cm.c
+++ b/trunk/ulp/dapl2/dapl/openib_cma/cm.c
@@ -167,6 +167,7 @@ dp_ib_cm_handle_t dapls_ib_cm_create(DAPL_EP *ep)
 \r
        /* create CM_ID, bind to local device, create QP */\r
        if (rdma_create_id(g_cm_events, &cm_id, (void *)conn, RDMA_PS_TCP)) {\r
+               dapl_os_lock_destroy(&conn->lock);\r
                dapl_os_free(conn, sizeof(*conn));\r
                return NULL;\r
        }\r
@@ -221,6 +222,7 @@ void dapls_ib_cm_free(dp_ib_cm_handle_t conn, DAPL_EP *ep)
                rdma_destroy_id(conn->cm_id);\r
        }\r
 \r
+       dapl_os_lock_destroy(&conn->lock);\r
        dapl_os_free(conn, sizeof(*conn));\r
 }\r
 \r
@@ -686,6 +688,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA * ia_ptr,
        /* create CM_ID, bind to local device, create QP */\r
        if (rdma_create_id\r
            (g_cm_events, &conn->cm_id, (void *)conn, RDMA_PS_TCP)) {\r
+               dapl_os_lock_destroy(&conn->lock);\r
                dapl_os_free(conn, sizeof(*conn));\r
                return (dapl_convert_errno(errno, "setup_listener"));\r
        }\r
@@ -734,6 +737,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA * ia_ptr,
 \r
       bail:\r
        rdma_destroy_id(conn->cm_id);\r
+       dapl_os_lock_destroy(&conn->lock);\r
        dapl_os_free(conn, sizeof(*conn));\r
        return dat_status;\r
 }\r

diff --git a/trunk/ulp/dapl2/dapl/openib_cma/device.c b/trunk/ulp/dapl2/dapl/openib_cma/device.c
index 32090fbef3c7bf97b695130a848a1de680394f71..c9d3756d71fc65b5af25680c96714e5e93f60855 100644 (file)
--- a/trunk/ulp/dapl2/dapl/openib_cma/device.c
+++ b/trunk/ulp/dapl2/dapl/openib_cma/device.c
@@ -502,6 +502,16 @@ DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr)
                dapl_os_sleep_usec(1000);\r
        }\r
 bail:\r
+       if (hca_ptr->ib_trans.ib_cq)\r
+               ibv_destroy_comp_channel(hca_ptr->ib_trans.ib_cq);\r
+\r
+       if (hca_ptr->ib_trans.ib_cq_empty) {\r
+               struct ibv_comp_channel *channel;\r
+               channel = hca_ptr->ib_trans.ib_cq_empty->channel;\r
+               ibv_destroy_cq(hca_ptr->ib_trans.ib_cq_empty);\r
+               ibv_destroy_comp_channel(channel);\r
+       }\r
+\r
        if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {\r
                if (rdma_destroy_id(hca_ptr->ib_trans.cm_id))\r
                        return (dapl_convert_errno(errno, "ib_close_device"));\r

diff --git a/trunk/ulp/dapl2/dapl/openib_scm/cm.c b/trunk/ulp/dapl2/dapl/openib_scm/cm.c
index 356f0fb5f72d4a28d882e4c198b21dd64af78f7f..03ba3acfd70922fd0c71450e4533801c328b88c1 100644 (file)
--- a/trunk/ulp/dapl2/dapl/openib_scm/cm.c
+++ b/trunk/ulp/dapl2/dapl/openib_scm/cm.c
@@ -317,6 +317,7 @@ void dapls_ib_cm_free(dp_ib_cm_handle_t cm_ptr, DAPL_EP *ep)
                        closesocket(cm_ptr->socket);\r
                }\r
                dapl_os_unlock(&cm_ptr->lock);\r
+               dapl_os_lock_destroy(&cm_ptr->lock);\r
                dapl_os_free(cm_ptr, sizeof(*cm_ptr));\r
                return;\r
        }\r
@@ -390,15 +391,17 @@ notify_thread:
 /* queue socket for processing CM work */\r
 static void dapli_cm_queue(struct ib_cm_handle *cm_ptr)\r
 {\r
+       DAPL_HCA *hca = cm_ptr->hca;\r
+\r
        /* add to work queue for cr thread processing */\r
        dapl_llist_init_entry((DAPL_LLIST_ENTRY *) & cm_ptr->entry);\r
-       dapl_os_lock(&cm_ptr->hca->ib_trans.lock);\r
-       dapl_llist_add_tail(&cm_ptr->hca->ib_trans.list,\r
+       dapl_os_lock(&hca->ib_trans.lock);\r
+       dapl_llist_add_tail(&hca->ib_trans.list,\r
                            (DAPL_LLIST_ENTRY *) & cm_ptr->entry, cm_ptr);\r
-       dapl_os_unlock(&cm_ptr->hca->ib_trans.lock);\r
+       dapl_os_unlock(&hca->ib_trans.lock);\r
 \r
        /* wakeup CM work thread */\r
-       send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0);\r
+       send(hca->ib_trans.scm[1], "w", sizeof "w", 0);\r
 }\r
 \r
 /*\r
@@ -1779,6 +1782,7 @@ void cr_thread(void *arg)
                                         inet_ntoa(((struct sockaddr_in *)\r
                                                &cr->msg.daddr.so)->sin_addr));\r
                                dapl_os_unlock(&cr->lock);\r
+                               dapl_os_lock_destroy(&cr->lock);\r
                                dapls_ib_cm_free(cr, cr->ep);\r
                                continue;\r
                        }\r

diff --git a/trunk/ulp/dapl2/dapl/openib_scm/device.c b/trunk/ulp/dapl2/dapl/openib_scm/device.c
index bb3893a72c3de5ade7fb323216f2f4a75e1f1872..04e992a8ef86dbb1bd317dfeb9da5c41a16d6b4e 100644 (file)
--- a/trunk/ulp/dapl2/dapl/openib_scm/device.c
+++ b/trunk/ulp/dapl2/dapl/openib_scm/device.c
@@ -504,6 +504,16 @@ DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr)
        }\r
 \r
 out:\r
+       if (hca_ptr->ib_trans.ib_cq)\r
+               ibv_destroy_comp_channel(hca_ptr->ib_trans.ib_cq);\r
+\r
+       if (hca_ptr->ib_trans.ib_cq_empty) {\r
+               struct ibv_comp_channel *channel;\r
+               channel = hca_ptr->ib_trans.ib_cq_empty->channel;\r
+               ibv_destroy_cq(hca_ptr->ib_trans.ib_cq_empty);\r
+               ibv_destroy_comp_channel(channel);\r
+       }\r
+\r
        if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {\r
                if (ibv_close_device(hca_ptr->ib_hca_handle))\r
                        return (dapl_convert_errno(errno, "ib_close_device"));\r

diff --git a/trunk/ulp/dapl2/dapl/openib_ucm/cm.c b/trunk/ulp/dapl2/dapl/openib_ucm/cm.c
index 100317bb85b7ee7821fb50f1fa1d27202c3c09fb..03fc75277bb994be192a2974163d26d428c39232 100644 (file)
--- a/trunk/ulp/dapl2/dapl/openib_ucm/cm.c
+++ b/trunk/ulp/dapl2/dapl/openib_ucm/cm.c
@@ -728,6 +728,7 @@ void dapls_ib_cm_free(dp_ib_cm_handle_t cm, DAPL_EP *ep)
        /* cleanup, never made it to work queue */\r
        if (cm->state == DCM_INIT) {\r
                dapl_os_unlock(&cm->lock);\r
+               dapl_os_lock_destroy(&cm->lock);\r
                dapl_os_free(cm, sizeof(*cm));\r
                return;\r
        }\r
@@ -1701,6 +1702,7 @@ dapls_ib_remove_conn_listener(IN DAPL_IA *ia, IN DAPL_SP *sp)
                cm->state = DCM_DESTROY;\r
                dapl_os_unlock(&cm->lock);\r
                ucm_dequeue_listen(cm);\r
+               dapl_os_lock_destroy(&cm->lock);\r
                dapl_os_free(cm, sizeof(*cm));\r
        }\r
        return DAT_SUCCESS;\r
@@ -1981,6 +1983,7 @@ void cm_thread(void *arg)
                                dapl_llist_remove_entry(&hca->ib_trans.list,\r
                                                        (DAPL_LLIST_ENTRY *)&cm->entry);\r
                                dapl_os_unlock(&cm->lock);\r
+                               dapl_os_lock_destroy(&cm->lock);\r
                                dapl_os_free(cm, sizeof(*cm));\r
                                continue;\r
                        }\r
@@ -2052,6 +2055,7 @@ void cm_thread(void *arg)
                                        &hca->ib_trans.list,\r
                                        (DAPL_LLIST_ENTRY *)&cm->entry);\r
                                dapl_os_unlock(&cm->lock);\r
+                               dapl_os_lock_destroy(&cm->lock);\r
                                dapl_os_free(cm, sizeof(*cm));\r
                                continue;\r
                        }\r

diff --git a/trunk/ulp/dapl2/dapl/openib_ucm/device.c b/trunk/ulp/dapl2/dapl/openib_ucm/device.c
index e890eefff056d73325c51858f05e0d4e82543b9f..a9cec73e91204719e28afaa16fcdc9037d94af6e 100644 (file)
--- a/trunk/ulp/dapl2/dapl/openib_ucm/device.c
+++ b/trunk/ulp/dapl2/dapl/openib_ucm/device.c
@@ -401,6 +401,16 @@ DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr)
        destroy_os_signal(hca_ptr);
        ucm_service_destroy(hca_ptr);
 
+       if (hca_ptr->ib_trans.ib_cq)\r
+               ibv_destroy_comp_channel(hca_ptr->ib_trans.ib_cq);\r
+\r
+       if (hca_ptr->ib_trans.ib_cq_empty) {\r
+               struct ibv_comp_channel *channel;\r
+               channel = hca_ptr->ib_trans.ib_cq_empty->channel;\r
+               ibv_destroy_cq(hca_ptr->ib_trans.ib_cq_empty);\r
+               ibv_destroy_comp_channel(channel);\r
+       }\r
+
        if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {
                if (ibv_close_device(hca_ptr->ib_hca_handle))
                        return (dapl_convert_errno(errno, "ib_close_device"));