librdmacm: allow for graceful cleanup of verbs

Have librdmacm release libibverbs resources when they are no longer in use.  This provides for more graceful cleanup of verbs resources.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1@2458 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/ulp/librdmacm/src/cma.cpp b/trunk/ulp/librdmacm/src/cma.cpp
index 7d31c30d0d1a75c451d546786bf3bb1f6ac07eb4..022d291087f04b853099874a979c795c434c4e96 100644 (file)
--- a/trunk/ulp/librdmacm/src/cma.cpp
+++ b/trunk/ulp/librdmacm/src/cma.cpp
@@ -90,8 +90,9 @@ struct cma_event {
 \r
 static struct cma_device *cma_dev_array;\r
 static int cma_dev_cnt;\r
+static DWORD ref;\r
 \r
-static int ucma_init(void)\r
+static int ucma_acquire(void)\r
 {\r
        struct ibv_device **dev_list = NULL;\r
        struct cma_device *cma_dev;\r
@@ -99,7 +100,7 @@ static int ucma_init(void)
        int i, ret, dev_cnt;\r
 \r
        EnterCriticalSection(&lock);\r
-       if (cma_dev_cnt) {\r
+       if (ref++) {\r
                goto out;\r
        }\r
 \r
@@ -141,7 +142,7 @@ static int ucma_init(void)
                cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;\r
        }\r
        ibv_free_device_list(dev_list);\r
-       cma_dev_cnt = dev_cnt;\r
+\r
 out:\r
        LeaveCriticalSection(&lock);\r
        return 0;\r
@@ -156,17 +157,34 @@ err3:
 err2:\r
        ibvw_release_windata(&windata, IBVW_WINDATA_VERSION);\r
 err1:\r
+       ref--;\r
        LeaveCriticalSection(&lock);\r
        return ret;\r
 }\r
 \r
+void ucma_release(void)\r
+{\r
+       int i;\r
+\r
+       EnterCriticalSection(&lock);\r
+       if (--ref == 0) {\r
+               for (i = 0; i < cma_dev_cnt; i++) {\r
+                       ibv_close_device(cma_dev_array[i].verbs);\r
+               }\r
+               delete cma_dev_array;\r
+               cma_dev_cnt = 0;\r
+               ibvw_release_windata(&windata, IBVW_WINDATA_VERSION);\r
+       }\r
+       LeaveCriticalSection(&lock);\r
+}\r
+\r
 __declspec(dllexport)\r
 struct ibv_context **rdma_get_devices(int *num_devices)\r
 {\r
        struct ibv_context **devs = NULL;\r
        int i;\r
 \r
-       if (!cma_dev_cnt && ucma_init()) {\r
+       if (ucma_acquire()) {\r
                goto out;\r
        }\r
 \r
@@ -190,6 +208,7 @@ __declspec(dllexport)
 void rdma_free_devices(struct ibv_context **list)\r
 {\r
        delete list;\r
+       ucma_release();\r
 }\r
 \r
 __declspec(dllexport)\r
@@ -197,7 +216,7 @@ struct rdma_event_channel *rdma_create_event_channel(void)
 {\r
        struct rdma_event_channel *channel;\r
 \r
-       if (!cma_dev_cnt && ucma_init()) {\r
+       if (ucma_acquire()) {\r
                return NULL;\r
        }\r
 \r
@@ -215,6 +234,7 @@ void rdma_destroy_event_channel(struct rdma_event_channel *channel)
 {\r
        CompChannelCleanup(&channel->channel);\r
        delete channel;\r
+       ucma_release();\r
 }\r
 \r
 __declspec(dllexport)\r
@@ -225,14 +245,15 @@ int rdma_create_id(struct rdma_event_channel *channel,
        struct cma_id_private *id_priv;\r
        HRESULT hr;\r
 \r
-       hr = cma_dev_cnt ? 0 : ucma_init();\r
+       hr = ucma_acquire();\r
        if (hr) {\r
                return hr;\r
        }\r
 \r
        id_priv = new struct cma_id_private;\r
        if (id_priv == NULL) {\r
-               return NULL;\r
+               hr = ENOMEM;\r
+               goto err1;\r
        }\r
 \r
        RtlZeroMemory(id_priv, sizeof(struct cma_id_private));\r
@@ -248,14 +269,16 @@ int rdma_create_id(struct rdma_event_channel *channel,
                hr = windata.prov->CreateDatagramEndpoint(&id_priv->id.ep.datagram);\r
        }\r
        if (FAILED(hr)) {\r
-               goto err;\r
+               goto err2;\r
        }\r
 \r
        *id = &id_priv->id;\r
        return 0;\r
 \r
-err:\r
+err2:\r
        delete id_priv;\r
+err1:\r
+       ucma_release();\r
        return hr;\r
 }\r
 \r
@@ -307,6 +330,7 @@ int rdma_destroy_id(struct rdma_cm_id *id)
                Sleep(0);\r
        }\r
        delete id_priv;\r
+       ucma_release();\r
        return 0;\r
 }\r
 \r
@@ -482,23 +506,33 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
        return 0;\r
 }\r
 \r
-__declspec(dllexport)\r
-int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)\r
+static int\r
+ucma_resolve_ibat_path(struct rdma_cm_id *id, int timeout_ms,\r
+                                          IBAT_PATH_BLOB *path)\r
 {\r
-       struct cma_id_private *id_priv;\r
-       IBAT_PATH_BLOB path;\r
        HRESULT hr;\r
 \r
        do {\r
-               hr = IBAT::Resolve(&id->route.addr.src_addr, &id->route.addr.dst_addr, &path);\r
-               if (hr != E_PENDING) {\r
+               hr = IBAT::Resolve(&id->route.addr.src_addr, &id->route.addr.dst_addr,\r
+                                                  path);\r
+               if (hr != E_PENDING || timeout_ms <= 0) {\r
                        break;\r
                }\r
                timeout_ms -= 10;\r
-               if (timeout_ms > 0)\r
-                       Sleep(10);\r
+               Sleep(10);\r
        } while (timeout_ms > 0);\r
 \r
+       return hr;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       IBAT_PATH_BLOB path;\r
+       HRESULT hr;\r
+\r
+       hr = ucma_resolve_ibat_path(id, timeout_ms, &path);\r
        if (FAILED(hr)) {\r
                return hr;\r
        }\r

diff --git a/trunk/ulp/librdmacm/src/cma.h b/trunk/ulp/librdmacm/src/cma.h
index fb65026b71aca1d888fa0b2fea67e88acbfffb6a..918a7936ee7dc34c2d1d68f7e6ff27eb75ef7a56 100644 (file)
--- a/trunk/ulp/librdmacm/src/cma.h
+++ b/trunk/ulp/librdmacm/src/cma.h
@@ -32,15 +32,18 @@
 #define CMA_H\r
 \r
 extern CRITICAL_SECTION lock;\r
+extern HANDLE heap;\r
+\r
+void ucma_cleanup();\r
 \r
 __inline void* __cdecl operator new(size_t size)\r
 {\r
-       return HeapAlloc(GetProcessHeap(), 0, size);\r
+       return HeapAlloc(heap, 0, size);\r
 }\r
 \r
 __inline void __cdecl operator delete(void *pObj)\r
 {\r
-       HeapFree(GetProcessHeap(), 0, pObj);\r
+       HeapFree(heap, 0, pObj);\r
 }\r
 \r
 #endif /* CMA_H */\r

diff --git a/trunk/ulp/librdmacm/src/cma_main.cpp b/trunk/ulp/librdmacm/src/cma_main.cpp
index 3521018bba6e69705592eee278db03e4d98c67f6..0084633de9649849964ce9bb33204a56f9235045 100644 (file)
--- a/trunk/ulp/librdmacm/src/cma_main.cpp
+++ b/trunk/ulp/librdmacm/src/cma_main.cpp
@@ -31,6 +31,7 @@
 #include "cma.h"\r
 \r
 CRITICAL_SECTION lock;\r
+HANDLE heap;\r
 \r
 BOOL WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)\r
 {\r
@@ -39,10 +40,15 @@ BOOL WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)
 \r
        switch (dwReason) {\r
        case DLL_PROCESS_ATTACH:\r
+               heap = HeapCreate(0, 0, 0);\r
+               if (heap == NULL) {\r
+                       return FALSE;\r
+               }\r
                InitializeCriticalSection(&lock);\r
                break;\r
        case DLL_PROCESS_DETACH:\r
                DeleteCriticalSection(&lock);\r
+               HeapDestroy(heap);\r
                break;\r
        default:\r
                break;\r