libibverbs: add reference counts to track CQ events

We need to track CQ events to block destroying CQ objects until all outstanding events have been acknowledged.  Otherwise, the caller could reference freed memory after calling ibv_get_cq_event.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1@2323 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/ulp/libibverbs/include/infiniband/verbs.h b/trunk/ulp/libibverbs/include/infiniband/verbs.h
index 700259e357c7ebaa69cdae42c4bfb9b06b817bdf..236ebe4212bd760a0de331cd0c9665c38e10419e 100644 (file)
--- a/trunk/ulp/libibverbs/include/infiniband/verbs.h
+++ b/trunk/ulp/libibverbs/include/infiniband/verbs.h
@@ -660,6 +660,8 @@ struct ibv_cq
        IWVCompletionQueue              *handle;\r
        int                                             cqe;\r
        COMP_ENTRY                              comp_entry;\r
+       LONG volatile                   notify_cnt;\r
+       LONG volatile                   ack_cnt;\r
 };\r
 \r
 struct ibv_ah\r

diff --git a/trunk/ulp/libibverbs/src/verbs.cpp b/trunk/ulp/libibverbs/src/verbs.cpp
index e5dffc6063a0d9e333f8c2c31fc6787a35976e5f..e577733c82b6c489783fdb8fa233b6d778fec54f 100644 (file)
--- a/trunk/ulp/libibverbs/src/verbs.cpp
+++ b/trunk/ulp/libibverbs/src/verbs.cpp
@@ -347,6 +347,8 @@ struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, void *cq_cont
        cq->context = context;\r
        cq->channel = channel;\r
        cq->cq_context = cq_context;\r
+       cq->notify_cnt = 0;\r
+       cq->ack_cnt = 0;\r
 \r
        entries = cqe;\r
        hr = context->cmd_if->CreateCompletionQueue(&entries, &cq->handle);\r
@@ -387,10 +389,14 @@ int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only)
        HRESULT hr;\r
 \r
        if (InterlockedCompareExchange(&cq->comp_entry.Busy, 1, 0) == 0) {\r
+               InterlockedIncrement(&cq->notify_cnt);\r
                hr = cq->handle->Notify(solicited_only ? WvCqSolicited : WvCqNextCompletion,\r
                                                                &cq->comp_entry.Overlap);\r
                if (SUCCEEDED(hr) || hr == WV_IO_PENDING) {\r
                        hr = 0;\r
+               } else {\r
+                       InterlockedExchange(&cq->comp_entry.Busy, 0);\r
+                       InterlockedDecrement(&cq->notify_cnt);\r
                }\r
        } else {\r
                hr = 0;\r
@@ -417,8 +423,13 @@ int ibv_destroy_cq(struct ibv_cq *cq)
 {\r
        cq->handle->CancelOverlappedRequests();\r
        if (cq->channel != NULL) {\r
-               CompEntryCancel(&cq->comp_entry);\r
+               if (CompEntryCancel(&cq->comp_entry)) {\r
+                       InterlockedIncrement(&cq->ack_cnt);\r
+               }\r
        }\r
+\r
+       while (cq->ack_cnt < cq->notify_cnt)\r
+               Sleep(0);\r
        cq->handle->Release();\r
        delete cq;\r
        return 0;\r
@@ -443,6 +454,7 @@ int ibv_get_cq_event(struct ibv_comp_channel *channel,
 __declspec(dllexport)\r
 void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)\r
 {\r
+       InterlockedExchangeAdd(&cq->ack_cnt, (LONG) nevents);\r
 }\r
 \r
 __declspec(dllexport)\r