From ee599441f309b023c2cc30b1bb2523981abd398b Mon Sep 17 00:00:00 2001 From: shefty Date: Sat, 26 Apr 2008 05:58:04 +0000 Subject: [PATCH] winverbs.sys: add initial support for CQ, SRQ, and QP This adds the starting implementation for CQ, SRQ, and QP support. Some code is restructured to relocate functions to different source files. This makes them easier to find and allows other functions to become static. The kernel driver now implements all synchronous verb related routines. The code is compile tested only, and does not yet implement any asynchronous routines (ones that involve the use of OVERLAPPED operations). Signed-off-by: Sean Hefty git-svn-id: svn://openib.tc.cornell.edu/gen1@1075 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- .../winverbs/core/winverbs/kernel/SOURCES | 3 + .../winverbs/core/winverbs/kernel/wv_cq.c | 240 +++++++++++ .../winverbs/core/winverbs/kernel/wv_cq.h | 21 +- .../winverbs/core/winverbs/kernel/wv_device.c | 189 ++++----- .../winverbs/core/winverbs/kernel/wv_device.h | 11 +- .../winverbs/core/winverbs/kernel/wv_driver.c | 192 +++++++-- .../winverbs/core/winverbs/kernel/wv_pd.c | 141 ++++++- .../winverbs/core/winverbs/kernel/wv_pd.h | 8 +- .../core/winverbs/kernel/wv_provider.c | 111 ++---- .../core/winverbs/kernel/wv_provider.h | 6 +- .../winverbs/core/winverbs/kernel/wv_qp.c | 376 ++++++++++++++++++ .../winverbs/core/winverbs/kernel/wv_qp.h | 24 +- .../winverbs/core/winverbs/kernel/wv_srq.c | 296 ++++++++++++++ .../winverbs/core/winverbs/kernel/wv_srq.h | 22 +- 14 files changed, 1392 insertions(+), 248 deletions(-) create mode 100644 branches/winverbs/core/winverbs/kernel/wv_cq.c create mode 100644 branches/winverbs/core/winverbs/kernel/wv_qp.c create mode 100644 branches/winverbs/core/winverbs/kernel/wv_srq.c diff --git a/branches/winverbs/core/winverbs/kernel/SOURCES b/branches/winverbs/core/winverbs/kernel/SOURCES index 2c874d84..42e8b43f 100644 --- a/branches/winverbs/core/winverbs/kernel/SOURCES +++ b/branches/winverbs/core/winverbs/kernel/SOURCES @@ -15,7 +15,10 @@ SOURCES = \ wv_driver.c \ wv_provider.c \ wv_device.c \ + wv_cq.c \ wv_pd.c \ + wv_srq.c \ + wv_qp.c \ index_list.c INCLUDES = ..;..\..\..\inc;..\..\..\inc\kernel;..\..\..\inc\user; diff --git a/branches/winverbs/core/winverbs/kernel/wv_cq.c b/branches/winverbs/core/winverbs/kernel/wv_cq.c new file mode 100644 index 00000000..806cf94a --- /dev/null +++ b/branches/winverbs/core/winverbs/kernel/wv_cq.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2008 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "wv_cq.h" +#include "wv_ioctl.h" + +void WvCqGet(WV_COMPLETION_QUEUE *pCq) +{ + InterlockedIncrement(&pCq->Ref); +} + +void WvCqPut(WV_COMPLETION_QUEUE *pCq) +{ + if (InterlockedDecrement(&pCq->Ref) == 0) { + KeSetEvent(&pCq->Event, 0, FALSE); + } +} + +WV_COMPLETION_QUEUE *WvCqAcquire(WV_PROVIDER *pProvider, UINT64 Id) +{ + WV_COMPLETION_QUEUE *cq; + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + cq = IndexListAt(&pProvider->CqIndex, (SIZE_T) Id); + if (cq != NULL && cq->hVerbsCq != NULL) { + WvCqGet(cq); + } else { + cq = NULL; + WvProviderEnableRemove(pProvider); + } + KeReleaseGuardedMutex(&pProvider->Lock); + + return cq; +} + +void WvCqRelease(WV_COMPLETION_QUEUE *pCq) +{ + WvProviderEnableRemove(pCq->pDevice->pProvider); + WvCqPut(pCq); +} + +static NTSTATUS WvCqAlloc(WV_DEVICE *pDevice, UINT32 *pSize, + WV_COMPLETION_QUEUE **ppCq, ci_umv_buf_t *pVerbsData) +{ + ib_api_status_t ib_status; + WV_COMPLETION_QUEUE *cq; + + cq = ExAllocatePoolWithTag(PagedPool, sizeof(WV_COMPLETION_QUEUE), 'qcvw'); + if (cq == NULL) { + return STATUS_NO_MEMORY; + } + + cq->Ref = 1; + KeInitializeEvent(&cq->Event, NotificationEvent, FALSE); + + ib_status = pDevice->pVerbs->create_cq(pDevice->hVerbsDevice, cq, + pSize, &cq->hVerbsCq, pVerbsData); + if (ib_status != IB_SUCCESS) { + goto err; + } + + cq->pVerbs = pDevice->pVerbs; + WvDeviceGet(pDevice); + *ppCq = cq; + return STATUS_SUCCESS; + +err: + ExFreePool(cq); + return STATUS_UNSUCCESSFUL; +} + +void WvCqCreate(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_ID *inid, *outid; + size_t inlen, outlen; + WV_DEVICE *dev; + WV_COMPLETION_QUEUE *cq; + NTSTATUS status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &inid, &inlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + + dev = WvDeviceAcquire(pProvider, inid->Id); + if (dev == NULL) { + status = STATUS_NO_SUCH_DEVICE; + goto err1; + } + + WvInitVerbsData(&verbsData, inid->VerbInfo, inlen - sizeof(WV_IO_ID), + outlen - sizeof(WV_IO_ID), inid + 1); + status = WvCqAlloc(dev, &inid->Data, &cq, &verbsData); + if (!NT_SUCCESS(status)) { + goto err2; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + outid->Id = IndexListInsertHead(&pProvider->CqIndex, cq); + if (outid->Id == 0) { + status = STATUS_NO_MEMORY; + goto err3; + } + InsertHeadList(&dev->CqList, &cq->Entry); + KeReleaseGuardedMutex(&pProvider->Lock); + + WvDeviceRelease(dev); + outid->Data = inid->Data; + outid->VerbInfo = verbsData.status; + WdfRequestCompleteWithInformation(Request, status, outlen); + return; + +err3: + KeReleaseGuardedMutex(&pProvider->Lock); + WvCqFree(cq); +err2: + WvDeviceRelease(dev); +err1: + WdfRequestComplete(Request, status); +} + +void WvCqDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_COMPLETION_QUEUE *cq; + UINT64 *id; + NTSTATUS status; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL); + if (!NT_SUCCESS(status)) { + goto out; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + cq = IndexListAt(&pProvider->CqIndex, (SIZE_T) *id); + if (cq == NULL) { + status = STATUS_NOT_FOUND; + } else if (cq->Ref > 1) { + status = STATUS_ACCESS_DENIED; + } else { + IndexListRemove(&pProvider->CqIndex, (SIZE_T) *id); + RemoveEntryList(&cq->Entry); + status = STATUS_SUCCESS; + } + KeReleaseGuardedMutex(&pProvider->Lock); + + if (NT_SUCCESS(status)) { + WvCqFree(cq); + } + WvProviderEnableRemove(pProvider); +out: + WdfRequestComplete(Request, status); +} + +void WvCqFree(WV_COMPLETION_QUEUE *pCq) +{ + if (InterlockedDecrement(&pCq->Ref) > 0) { + KeWaitForSingleObject(&pCq->Event, Executive, KernelMode, FALSE, NULL); + } + + if (pCq->hVerbsCq != NULL) { + pCq->pVerbs->destroy_cq(pCq->hVerbsCq); + } + + WvDevicePut(pCq->pDevice); + ExFreePool(pCq); +} + +void WvCqResize(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_ID *inid, *outid; + size_t inlen, outlen, len = 0; + WV_COMPLETION_QUEUE *cq; + NTSTATUS status; + ib_api_status_t ib_status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &inid, &inlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + + cq = WvCqAcquire(pProvider, inid->Id); + if (cq == NULL) { + status = STATUS_NOT_FOUND; + goto complete; + } + + WvInitVerbsData(&verbsData, inid->VerbInfo, inlen - sizeof(WV_IO_ID), + outlen - sizeof(WV_IO_ID), inid + 1); + ib_status = cq->pVerbs->resize_cq(cq->hVerbsCq, &inid->Data, &verbsData); + WvCqRelease(cq); + + if (ib_status != IB_SUCCESS) { + status = STATUS_UNSUCCESSFUL; + } + + len = outlen; + outid->Data = inid->Data; + outid->VerbInfo = verbsData.status; + +complete: + WdfRequestCompleteWithInformation(Request, status, len); +} diff --git a/branches/winverbs/core/winverbs/kernel/wv_cq.h b/branches/winverbs/core/winverbs/kernel/wv_cq.h index 4469f291..dbc5f55d 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_cq.h +++ b/branches/winverbs/core/winverbs/kernel/wv_cq.h @@ -32,19 +32,36 @@ #ifndef _WV_CQ_H_ #define _WV_CQ_H_ +#include #include #include #include + #include "wv_device.h" +#include "wv_provider.h" typedef struct _WV_COMPLETION_QUEUE { WV_DEVICE *pDevice; ci_interface_t *pVerbs; ib_cq_handle_t hVerbsCq; - UINT64 Id; - LONG nRef; + LIST_ENTRY Entry; + + KEVENT Event; + LONG Ref; } WV_COMPLETION_QUEUE; + +void WvCqCreate(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvCqDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvCqFree(WV_COMPLETION_QUEUE *pCq); + +WV_COMPLETION_QUEUE *WvCqAcquire(WV_PROVIDER *pProvider, UINT64 Id); +void WvCqRelease(WV_COMPLETION_QUEUE *pCq); +void WvCqGet(WV_COMPLETION_QUEUE *pCq); +void WvCqPut(WV_COMPLETION_QUEUE *pCq); + +void WvCqResize(WV_PROVIDER *pProvider, WDFREQUEST Request); + #endif //_WV_CQ_H_ diff --git a/branches/winverbs/core/winverbs/kernel/wv_device.c b/branches/winverbs/core/winverbs/kernel/wv_device.c index 647f3448..ca10fcb5 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_device.c +++ b/branches/winverbs/core/winverbs/kernel/wv_device.c @@ -29,6 +29,7 @@ #include "wv_device.h" #include "wv_pd.h" +#include "wv_cq.h" #include "wv_ioctl.h" void WvDeviceGet(WV_DEVICE *pDevice) @@ -67,7 +68,7 @@ void WvDeviceRelease(WV_DEVICE *pDevice) WvDevicePut(pDevice); } -WV_DEVICE *WvDeviceAlloc(WV_PROVIDER *pProvider) +static WV_DEVICE *WvDeviceAlloc(WV_PROVIDER *pProvider) { WV_DEVICE *dev; @@ -81,6 +82,7 @@ WV_DEVICE *WvDeviceAlloc(WV_PROVIDER *pProvider) dev->hVerbsDevice = NULL; dev->Ref = 1; InitializeListHead(&dev->PdList); + InitializeListHead(&dev->CqList); KeInitializeEvent(&dev->Event, NotificationEvent, FALSE); dev->pProvider = pProvider; @@ -88,7 +90,8 @@ WV_DEVICE *WvDeviceAlloc(WV_PROVIDER *pProvider) return dev; } -NTSTATUS WvDeviceInit(WV_DEVICE *pDevice, UINT64 Guid, ci_umv_buf_t *pVerbsData) +static NTSTATUS WvDeviceInit(WV_DEVICE *pDevice, UINT64 Guid, + ci_umv_buf_t *pVerbsData) { WV_RDMA_DEVICE *dev; ib_api_status_t ib_status; @@ -114,6 +117,92 @@ err: return STATUS_UNSUCCESSFUL; } +void WvDeviceOpen(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_ID *inid, *outid; + size_t inlen, outlen; + WV_DEVICE *dev; + NTSTATUS status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &inid, &inlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + + dev = WvDeviceAlloc(pProvider); + if (dev == NULL) { + status = STATUS_NO_MEMORY; + goto err1; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + KeReleaseGuardedMutex(&pProvider->Lock); + + WvInitVerbsData(&verbsData, inid->VerbInfo, inlen - sizeof(WV_IO_ID), + outlen - sizeof(WV_IO_ID), inid + 1); + status = WvDeviceInit(dev, inid->Id, &verbsData); + if (!NT_SUCCESS(status)) { + goto err2; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + outid->Id = IndexListInsertHead(&pProvider->DevIndex, dev); + if (outid->Id == 0) { + status = STATUS_NO_MEMORY; + goto err2; + } + KeReleaseGuardedMutex(&pProvider->Lock); + + WvProviderEnableRemove(pProvider); + outid->VerbInfo = verbsData.status; + WdfRequestCompleteWithInformation(Request, status, outlen); + return; + +err2: + WvDeviceFree(dev); + WvProviderEnableRemove(pProvider); +err1: + WdfRequestComplete(Request, status); +} + +void WvDeviceClose(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_DEVICE *dev; + UINT64 *id; + NTSTATUS status; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL); + if (!NT_SUCCESS(status)) { + goto out; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + dev = IndexListAt(&pProvider->DevIndex, (SIZE_T) *id); + if (dev == NULL) { + status = STATUS_NO_SUCH_DEVICE; + } else if (dev->Ref > 1) { + status = STATUS_ACCESS_DENIED; + } else { + IndexListRemove(&pProvider->DevIndex, (SIZE_T) *id); + status = STATUS_SUCCESS; + } + KeReleaseGuardedMutex(&pProvider->Lock); + + if (NT_SUCCESS(status)) { + WvDeviceFree(dev); + } + WvProviderEnableRemove(pProvider); +out: + WdfRequestComplete(Request, status); +} + void WvDeviceFree(WV_DEVICE *pDevice) { if (InterlockedDecrement(&pDevice->Ref) > 0) { @@ -134,6 +223,7 @@ void WvDeviceRemoveHandler(WV_DEVICE *pDevice) { LIST_ENTRY *entry; WV_PROTECTION_DOMAIN *pd; + WV_COMPLETION_QUEUE *cq; for (entry = pDevice->PdList.Flink; entry != &pDevice->PdList; entry = entry->Flink) { @@ -141,6 +231,14 @@ void WvDeviceRemoveHandler(WV_DEVICE *pDevice) WvPdRemoveHandler(pd); } + for (entry = pDevice->CqList.Flink; entry != &pDevice->CqList; + entry = entry->Flink) { + cq = CONTAINING_RECORD(entry, WV_COMPLETION_QUEUE, Entry); + pDevice->pVerbs->destroy_cq(cq->hVerbsCq); + cq->hVerbsCq = NULL; + cq->pVerbs = NULL; + } + pDevice->pVerbs->um_close_ca(pDevice->pDevice->hDevice, pDevice->hVerbsDevice); WvRdmaDevicePut(pDevice->pDevice); @@ -498,90 +596,3 @@ free: complete: WdfRequestCompleteWithInformation(Request, status, outlen); } - -void WvPdAllocate(WV_PROVIDER *pProvider, WDFREQUEST Request) -{ - WV_IO_ID *inid, *outid; - size_t inlen, outlen; - WV_DEVICE *dev; - WV_PROTECTION_DOMAIN *pd; - NTSTATUS status; - ci_umv_buf_t verbsData; - - status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &inid, &inlen); - if (!NT_SUCCESS(status)) { - goto err1; - } - status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); - if (!NT_SUCCESS(status)) { - goto err1; - } - - dev = WvDeviceAcquire(pProvider, inid->Id); - if (dev == NULL) { - status = STATUS_NO_SUCH_DEVICE; - goto err1; - } - - WvInitVerbsData(&verbsData, inid->VerbInfo, inlen - sizeof(WV_IO_ID), - outlen - sizeof(WV_IO_ID), inid + 1); - status = WvPdAlloc(dev, &pd, &verbsData); - if (!NT_SUCCESS(status)) { - goto err2; - } - - KeAcquireGuardedMutex(&pProvider->Lock); - outid->Id= IndexListInsertHead(&pProvider->PdIndex, pd); - if (outid->Id == 0) { - status = STATUS_NO_MEMORY; - goto err3; - } - InsertHeadList(&dev->PdList, &pd->Entry); - KeReleaseGuardedMutex(&pProvider->Lock); - - WvDeviceRelease(dev); - outid->VerbInfo = verbsData.status; - WdfRequestCompleteWithInformation(Request, status, outlen); - return; - -err3: - KeReleaseGuardedMutex(&pProvider->Lock); - WvPdFree(pd); -err2: - WvDeviceRelease(dev); -err1: - WdfRequestComplete(Request, status); -} - -void WvPdDeallocate(WV_PROVIDER *pProvider, WDFREQUEST Request) -{ - WV_PROTECTION_DOMAIN *pd; - UINT64 *id; - NTSTATUS status; - - status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL); - if (!NT_SUCCESS(status)) { - goto out; - } - - KeAcquireGuardedMutex(&pProvider->Lock); - WvProviderDisableRemove(pProvider); - pd = IndexListAt(&pProvider->PdIndex, (SIZE_T) id); - if (pd == NULL) { - status = STATUS_NOT_FOUND; - } else if (pd->Ref > 1) { - status = STATUS_ACCESS_DENIED; - } else { - IndexListRemove(&pProvider->PdIndex, (SIZE_T) id); - RemoveEntryList(&pd->Entry); - status = STATUS_SUCCESS; - } - KeReleaseGuardedMutex(&pProvider->Lock); - - if (NT_SUCCESS(status)) { - WvPdFree(pd); - } - WvProviderEnableRemove(pProvider); -out: - WdfRequestComplete(Request, status); -} diff --git a/branches/winverbs/core/winverbs/kernel/wv_device.h b/branches/winverbs/core/winverbs/kernel/wv_device.h index 943bf84a..45a16944 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_device.h +++ b/branches/winverbs/core/winverbs/kernel/wv_device.h @@ -47,8 +47,10 @@ typedef struct _WV_DEVICE ci_interface_t *pVerbs; LIST_ENTRY Entry; ib_ca_handle_t hVerbsDevice; + LIST_ENTRY PdList; - SIZE_T Id; + LIST_ENTRY CqList; + KEVENT Event; LONG Ref; @@ -59,8 +61,8 @@ void WvDeviceRelease(WV_DEVICE *pDevice); void WvDeviceGet(WV_DEVICE *pDevice); void WvDevicePut(WV_DEVICE *pDevice); -WV_DEVICE *WvDeviceAlloc(WV_PROVIDER *pProvider); -NTSTATUS WvDeviceInit(WV_DEVICE *pDevice, UINT64 Guid, ci_umv_buf_t *pVerbsData); +void WvDeviceOpen(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvDeviceClose(WV_PROVIDER *pProvider, WDFREQUEST Request); void WvDeviceFree(WV_DEVICE *pDevice); void WvDeviceRemoveHandler(WV_DEVICE *pDevice); @@ -69,7 +71,4 @@ void WvDevicePortQuery(WV_PROVIDER *pProvider, WDFREQUEST Request); void WvDeviceGidQuery(WV_PROVIDER *pProvider, WDFREQUEST Request); void WvDevicePkeyQuery(WV_PROVIDER *pProvider, WDFREQUEST Request); -void WvPdAllocate(WV_PROVIDER *pProvider, WDFREQUEST Request); -void WvPdDeallocate(WV_PROVIDER *pProvider, WDFREQUEST Request); - #endif // __WV_DEVICE_H_ diff --git a/branches/winverbs/core/winverbs/kernel/wv_driver.c b/branches/winverbs/core/winverbs/kernel/wv_driver.c index a8a19a31..1f052fbe 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_driver.c +++ b/branches/winverbs/core/winverbs/kernel/wv_driver.c @@ -39,6 +39,7 @@ #include "wv_provider.h" #include "wv_device.h" #include "wv_pd.h" +#include "wv_srq.h" #include "wv_cq.h" #include "wv_srq.h" #include "wv_qp.h" @@ -164,6 +165,7 @@ static VOID WvIoDeviceControl(WDFQUEUE Queue, WDFREQUEST Request, file = WdfRequestGetFileObject(Request); prov = WvProviderGetContext(file); + // TODO: verify this compiles as a jump table, or use function pointers switch (IoControlCode) { case WV_IOCTL_GUID_QUERY: WvGuidQuery(Request); @@ -189,8 +191,14 @@ static VOID WvIoDeviceControl(WDFQUEUE Queue, WDFREQUEST Request, case WV_IOCTL_DEVICE_PKEY_QUERY: WvDevicePkeyQuery(prov, Request); break; - //case WV_IOCTL_DEVICE_NOTIFY:WvDeviceNotify;break; - //case WV_IOCTL_DEVICE_CANCEL:WvDeviceCancel;break; + case WV_IOCTL_DEVICE_NOTIFY: + //WvDeviceNotify(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_DEVICE_CANCEL: + //WvDeviceCancel(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; case WV_IOCTL_PD_ALLOCATE: WvPdAllocate(prov, Request); break; @@ -215,49 +223,147 @@ static VOID WvIoDeviceControl(WDFQUEUE Queue, WDFREQUEST Request, case WV_IOCTL_AH_DESTROY: WvAhDestroy(prov, Request); break; - //case WV_IOCTL_CQ_CREATE:WvCqCreate;break; - //case WV_IOCTL_CQ_DESTROY:WvCqDestroy;break; - //case WV_IOCTL_CQ_RESIZE:WvCqResize;break; - //case WV_IOCTL_CQ_NOTIFY:WvCqNotify;break; - //case WV_IOCTL_CQ_BATCH_NOTIFY:WvCqBatchNotify;break; - //case WV_IOCTL_CQ_CANCEL:WvCqCancel;break; - - //case WV_IOCTL_SRQ_CREATE:WvSrqCreate;break; - //case WV_IOCTL_SRQ_DESTROY:WvSrqDestroy;break; - //case WV_IOCTL_SRQ_QUERY:WvSrqQuery;break; - //case WV_IOCTL_SRQ_MODIFY:WvSrqModify;break; - //case WV_IOCTL_SRQ_NOTIFY:WvSrqNotify;break; - //case WV_IOCTL_SRQ_CANCEL:WvSrqCancel;break; - - //case WV_IOCTL_QP_CREATE:WvQpCreate;break; - //case WV_IOCTL_QP_DESTROY:WvQpDestroy;break; - //case WV_IOCTL_QP_QUERY:WvQpQuery;break; - //case WV_IOCTL_QP_MODIFY:WvQpModify;break; - //case WV_IOCTL_QP_ATTACH:WvQpAttach;break; - //case WV_IOCTL_QP_DETACH:WvQpDetach;break; - //case WV_IOCTL_QP_CANCEL:WvQpCancel;break; - - //case WV_IOCTL_ADDRESS_QUERY:WvAddressQuery;break; - //case WV_IOCTL_EP_CREATE:WvEpCreate;break; - //case WV_IOCTL_EP_DESTROY:WvEpDestroy;break; - //case WV_IOCTL_EP_BIND:WvEpBind;break; - //case WV_IOCTL_EP_REJECT:WvEpReject;break; - //case WV_IOCTL_EP_CONNECT:WvEpConnect;break; - //case WV_IOCTL_EP_ACCEPT:WvEpAccept;break; - //case WV_IOCTL_EP_DISCONNECT:WvEpDisconnect;break; - //case WV_IOCTL_EP_DISCONNECT_NOTIFY:WvEpDisconnectNotify;break; - //case WV_IOCTL_EP_QUERY:WvEpQuery;break; - //case WV_IOCTL_EP_LOOKUP:WvEpLookup;break; - //case WV_IOCTL_EP_MULTICAST_JOIN:WvEpMulticastJoin;break; - //case WV_IOCTL_EP_MULTICAST_LEAVE:WvEpMulticastLeave;break; - //case WV_IOCTL_EP_CANCEL:WvEpCancel;break; - //case WV_IOCTL_LISTEN:WvListen;break; - //case WV_IOCTL_LISTEN_DESTROY:WvListenDestroy;break; - //case WV_IOCTL_LISTEN_GET_REQUEST:WvListenGetRequest;break; - //case WV_IOCTL_LISTEN_CANCEL:WvListenCancel;break; - default: + case WV_IOCTL_CQ_CREATE: + WvCqCreate(prov, Request); + break; + case WV_IOCTL_CQ_DESTROY: + WvCqDestroy(prov, Request); + break; + case WV_IOCTL_CQ_RESIZE: + WvCqResize(prov, Request); + break; + case WV_IOCTL_CQ_NOTIFY: + //WvCqNotify(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_CQ_BATCH_NOTIFY: + //WvCqBatchNotify(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_CQ_CANCEL: + //WvCqCancel(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_SRQ_CREATE: + WvSrqCreate(prov, Request); + break; + case WV_IOCTL_SRQ_DESTROY: + WvSrqDestroy(prov, Request); + break; + case WV_IOCTL_SRQ_QUERY: + WvSrqQuery(prov, Request); + break; + case WV_IOCTL_SRQ_MODIFY: + WvSrqModify(prov, Request); + break; + case WV_IOCTL_SRQ_NOTIFY: + //WvSrqNotify(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_SRQ_CANCEL: + //WvSrqCancel(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_QP_CREATE: + WvQpCreate(prov, Request); + break; + case WV_IOCTL_QP_DESTROY: + WvQpDestroy(prov, Request); + break; + case WV_IOCTL_QP_QUERY: + WvQpQuery(prov, Request); + break; + case WV_IOCTL_QP_MODIFY: + //WvQpModify(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_QP_ATTACH: + //WvQpAttach(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_QP_DETACH: + //WvQpDetach(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_QP_CANCEL: + //WvQpCancel(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_ADDRESS_QUERY: + //WvAddressQuery(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_CREATE: + //WvEpCreate(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_DESTROY: + //WvEpDestroy(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_BIND: + //WvEpBind(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_REJECT: + //WvEpReject(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_CONNECT: + //WvEpConnect(prov, Request); WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); break; + case WV_IOCTL_EP_ACCEPT: + //WvEpAccept(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_DISCONNECT: + //WvEpDisconnect(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_DISCONNECT_NOTIFY: + //WvEpDisconnectNotify(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_QUERY: + //WvEpQuery(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_LOOKUP: + //WvEpLookup(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_MULTICAST_JOIN: + //WvEpMulticastJoin(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_MULTICAST_LEAVE: + //WvEpMulticastLeave(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_EP_CANCEL: + //WvEpCancel(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_LISTEN: + //WvListen(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_LISTEN_DESTROY: + //WvListenDestroy(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_LISTEN_GET_REQUEST: + //WvListenGetRequest(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + case WV_IOCTL_LISTEN_CANCEL: + //WvListenCancel(prov, Request); + WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED); + break; + default: + WdfRequestComplete(Request, STATUS_PROCEDURE_NOT_FOUND); + break; } } diff --git a/branches/winverbs/core/winverbs/kernel/wv_pd.c b/branches/winverbs/core/winverbs/kernel/wv_pd.c index 09a6fe11..d20cd7a2 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_pd.c +++ b/branches/winverbs/core/winverbs/kernel/wv_pd.c @@ -28,6 +28,8 @@ */ #include "wv_pd.h" +#include "wv_srq.h" +#include "wv_qp.h" #include "wv_ioctl.h" void WvPdGet(WV_PROTECTION_DOMAIN *pPd) @@ -66,19 +68,21 @@ void WvPdRelease(WV_PROTECTION_DOMAIN *pPd) WvPdPut(pPd); } -NTSTATUS WvPdAlloc(WV_DEVICE *pDevice, WV_PROTECTION_DOMAIN **ppPd, - ci_umv_buf_t *pVerbsData) +static NTSTATUS WvPdAlloc(WV_DEVICE *pDevice, WV_PROTECTION_DOMAIN **ppPd, + ci_umv_buf_t *pVerbsData) { ib_api_status_t ib_status; WV_PROTECTION_DOMAIN *pd; - pd = ExAllocatePoolWithTag(PagedPool, sizeof(WV_PROTECTION_DOMAIN), 'apvw'); + pd = ExAllocatePoolWithTag(PagedPool, sizeof(WV_PROTECTION_DOMAIN), 'dpvw'); if (pd == NULL) { return STATUS_NO_MEMORY; } pd->Ref = 1; KeInitializeEvent(&pd->Event, NotificationEvent, FALSE); + InitializeListHead(&pd->QpList); + InitializeListHead(&pd->SrqList); InitializeListHead(&pd->MwList); InitializeListHead(&pd->AhList); cl_qmap_init(&pd->MrMap); @@ -101,6 +105,93 @@ err: return STATUS_UNSUCCESSFUL; } +void WvPdAllocate(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_ID *inid, *outid; + size_t inlen, outlen; + WV_DEVICE *dev; + WV_PROTECTION_DOMAIN *pd; + NTSTATUS status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &inid, &inlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + + dev = WvDeviceAcquire(pProvider, inid->Id); + if (dev == NULL) { + status = STATUS_NO_SUCH_DEVICE; + goto err1; + } + + WvInitVerbsData(&verbsData, inid->VerbInfo, inlen - sizeof(WV_IO_ID), + outlen - sizeof(WV_IO_ID), inid + 1); + status = WvPdAlloc(dev, &pd, &verbsData); + if (!NT_SUCCESS(status)) { + goto err2; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + outid->Id = IndexListInsertHead(&pProvider->PdIndex, pd); + if (outid->Id == 0) { + status = STATUS_NO_MEMORY; + goto err3; + } + InsertHeadList(&dev->PdList, &pd->Entry); + KeReleaseGuardedMutex(&pProvider->Lock); + + WvDeviceRelease(dev); + outid->VerbInfo = verbsData.status; + WdfRequestCompleteWithInformation(Request, status, outlen); + return; + +err3: + KeReleaseGuardedMutex(&pProvider->Lock); + WvPdFree(pd); +err2: + WvDeviceRelease(dev); +err1: + WdfRequestComplete(Request, status); +} + +void WvPdDeallocate(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_PROTECTION_DOMAIN *pd; + UINT64 *id; + NTSTATUS status; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL); + if (!NT_SUCCESS(status)) { + goto out; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + pd = IndexListAt(&pProvider->PdIndex, (SIZE_T) *id); + if (pd == NULL) { + status = STATUS_NOT_FOUND; + } else if (pd->Ref > 1) { + status = STATUS_ACCESS_DENIED; + } else { + IndexListRemove(&pProvider->PdIndex, (SIZE_T) *id); + RemoveEntryList(&pd->Entry); + status = STATUS_SUCCESS; + } + KeReleaseGuardedMutex(&pProvider->Lock); + + if (NT_SUCCESS(status)) { + WvPdFree(pd); + } + WvProviderEnableRemove(pProvider); +out: + WdfRequestComplete(Request, status); +} + void WvPdFree(WV_PROTECTION_DOMAIN *pPd) { WV_MEMORY_REGION *mr; @@ -131,8 +222,25 @@ void WvPdFree(WV_PROTECTION_DOMAIN *pPd) void WvPdRemoveHandler(WV_PROTECTION_DOMAIN *pPd) { - WV_MEMORY_REGION *mr; - cl_map_item_t *item; + WV_QUEUE_PAIR *qp; + WV_SHARED_RECEIVE_QUEUE *srq; + WV_MEMORY_REGION *mr; + WV_ADDRESS_HANDLE *ah; + WV_MEMORY_WINDOW *mw; + cl_map_item_t *item; + LIST_ENTRY *entry; + + for (entry = pPd->MwList.Flink; entry != &pPd->MwList; entry = entry->Flink) { + mw = CONTAINING_RECORD(entry, WV_MEMORY_WINDOW, Entry); + pPd->pVerbs->destroy_mw(mw->hVerbsMw); + mw->hVerbsMw = NULL; + } + + for (entry = pPd->AhList.Flink; entry != &pPd->AhList; entry = entry->Flink) { + ah = CONTAINING_RECORD(entry, WV_ADDRESS_HANDLE, Entry); + pPd->pVerbs->destroy_av(ah->hVerbsAh); + ah->hVerbsAh = NULL; + } for (item = cl_qmap_head(&pPd->MrMap); item != cl_qmap_end(&pPd->MrMap); item = cl_qmap_next(item)) { @@ -141,6 +249,19 @@ void WvPdRemoveHandler(WV_PROTECTION_DOMAIN *pPd) mr->hVerbsMr = NULL; } + for (entry = pPd->QpList.Flink; entry != &pPd->QpList; entry = entry->Flink) { + qp = CONTAINING_RECORD(entry, WV_QUEUE_PAIR, Entry); + pPd->pVerbs->destroy_qp(qp->hVerbsQp, 0); + qp->hVerbsQp = NULL; + } + + for (entry = pPd->SrqList.Flink; entry != &pPd->SrqList; entry = entry->Flink) { + srq = CONTAINING_RECORD(entry, WV_SHARED_RECEIVE_QUEUE, Entry); + pPd->pVerbs->destroy_srq(srq->hVerbsSrq); + srq->hVerbsSrq = NULL; + srq->pVerbs = NULL; + } + pPd->pVerbs->deallocate_pd(pPd->hVerbsPd); pPd->pVerbs = NULL; pPd->hVerbsPd = NULL; @@ -331,11 +452,11 @@ void WvMwDeallocate(WV_PROVIDER *pProvider, WDFREQUEST Request) KeAcquireGuardedMutex(&pProvider->Lock); WvProviderDisableRemove(pProvider); - mw = IndexListAt(&pProvider->MwIndex, (SIZE_T) id); + mw = IndexListAt(&pProvider->MwIndex, (SIZE_T) *id); if (mw == NULL) { status = STATUS_NOT_FOUND; } else { - IndexListRemove(&pProvider->MwIndex, (SIZE_T) id); + IndexListRemove(&pProvider->MwIndex, (SIZE_T) *id); RemoveEntryList(&mw->Entry); status = STATUS_SUCCESS; } @@ -362,7 +483,7 @@ void WvMwFree(WV_MEMORY_WINDOW *pMw) static void WvVerbsConvertAv(ib_av_attr_t *pVerbsAv, WV_IO_AV *pAv) { if (pAv->GrhValid) { - RtlCopyMemory(&pVerbsAv->grh, &pAv, sizeof pVerbsAv->grh); + RtlCopyMemory(&pVerbsAv->grh, pAv, sizeof pVerbsAv->grh); pVerbsAv->grh_valid = 1; } else { pVerbsAv->grh_valid = 0; @@ -457,11 +578,11 @@ void WvAhDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request) KeAcquireGuardedMutex(&pProvider->Lock); WvProviderDisableRemove(pProvider); - ah = IndexListAt(&pProvider->AhIndex, (SIZE_T) id); + ah = IndexListAt(&pProvider->AhIndex, (SIZE_T) *id); if (ah == NULL) { status = STATUS_NOT_FOUND; } else { - IndexListRemove(&pProvider->AhIndex, (SIZE_T) id); + IndexListRemove(&pProvider->AhIndex, (SIZE_T) *id); RemoveEntryList(&ah->Entry); status = STATUS_SUCCESS; } diff --git a/branches/winverbs/core/winverbs/kernel/wv_pd.h b/branches/winverbs/core/winverbs/kernel/wv_pd.h index 74e06769..e0cd9b11 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_pd.h +++ b/branches/winverbs/core/winverbs/kernel/wv_pd.h @@ -47,6 +47,8 @@ typedef struct _WV_PROTECTION_DOMAIN ib_pd_handle_t hVerbsPd; LIST_ENTRY Entry; + LIST_ENTRY QpList; + LIST_ENTRY SrqList; LIST_ENTRY MwList; LIST_ENTRY AhList; KGUARDED_MUTEX Lock; @@ -57,13 +59,13 @@ typedef struct _WV_PROTECTION_DOMAIN } WV_PROTECTION_DOMAIN; -struct _WV_PROTECTION_DOMAIN *WvPdAcquire(WV_PROVIDER *pProvider, UINT64 Id); +WV_PROTECTION_DOMAIN *WvPdAcquire(WV_PROVIDER *pProvider, UINT64 Id); void WvPdRelease(WV_PROTECTION_DOMAIN *pPd); void WvPdGet(WV_PROTECTION_DOMAIN *pPd); void WvPdPut(WV_PROTECTION_DOMAIN *pPd); -NTSTATUS WvPdAlloc(WV_DEVICE *pDevice, WV_PROTECTION_DOMAIN **ppPd, - ci_umv_buf_t *pVerbsData); +void WvPdAllocate(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvPdDeallocate(WV_PROVIDER *pProvider, WDFREQUEST Request); void WvPdFree(WV_PROTECTION_DOMAIN *pPd); void WvPdRemoveHandler(WV_PROTECTION_DOMAIN *pPd); diff --git a/branches/winverbs/core/winverbs/kernel/wv_provider.c b/branches/winverbs/core/winverbs/kernel/wv_provider.c index a3c8d894..04116418 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_provider.c +++ b/branches/winverbs/core/winverbs/kernel/wv_provider.c @@ -35,6 +35,9 @@ #include "wv_provider.h" #include "wv_device.h" #include "wv_pd.h" +#include "wv_srq.h" +#include "wv_cq.h" +#include "wv_qp.h" void WvProviderGet(WV_PROVIDER *pProvider) { @@ -51,7 +54,10 @@ void WvProviderPut(WV_PROVIDER *pProvider) void WvProviderInit(WV_PROVIDER *pProvider) { IndexListInit(&pProvider->DevIndex); + IndexListInit(&pProvider->CqIndex); IndexListInit(&pProvider->PdIndex); + IndexListInit(&pProvider->SrqIndex); + IndexListInit(&pProvider->QpIndex); IndexListInit(&pProvider->MwIndex); IndexListInit(&pProvider->AhIndex); @@ -69,7 +75,10 @@ void WvProviderInit(WV_PROVIDER *pProvider) void WvProviderCleanup(WV_PROVIDER *pProvider) { WV_DEVICE *dev; + WV_COMPLETION_QUEUE *cq; WV_PROTECTION_DOMAIN *pd; + WV_SHARED_RECEIVE_QUEUE *srq; + WV_QUEUE_PAIR *qp; WV_MEMORY_WINDOW *mw; WV_ADDRESS_HANDLE *ah; @@ -81,10 +90,22 @@ void WvProviderCleanup(WV_PROVIDER *pProvider) RemoveEntryList(&mw->Entry); WvMwFree(mw); } + while ((qp = IndexListRemoveHead(&pProvider->QpIndex)) != NULL) { + RemoveEntryList(&qp->Entry); + WvQpFree(qp); + } + while ((srq = IndexListRemoveHead(&pProvider->SrqIndex)) != NULL) { + RemoveEntryList(&srq->Entry); + WvSrqFree(srq); + } while ((pd = IndexListRemoveHead(&pProvider->PdIndex)) != NULL) { RemoveEntryList(&pd->Entry); WvPdFree(pd); } + while ((cq = IndexListRemoveHead(&pProvider->CqIndex)) != NULL) { + RemoveEntryList(&cq->Entry); + WvCqFree(cq); + } while ((dev = IndexListRemoveHead(&pProvider->DevIndex)) != NULL) { WvDeviceFree(dev); } @@ -95,7 +116,10 @@ void WvProviderCleanup(WV_PROVIDER *pProvider) IndexListDestroy(&pProvider->AhIndex); IndexListDestroy(&pProvider->MwIndex); + IndexListDestroy(&pProvider->QpIndex); + IndexListDestroy(&pProvider->SrqIndex); IndexListDestroy(&pProvider->PdIndex); + IndexListDestroy(&pProvider->CqIndex); IndexListDestroy(&pProvider->DevIndex); } @@ -180,90 +204,3 @@ void WvProviderRemoveHandler(WV_PROVIDER *pProvider, WV_RDMA_DEVICE *pDevice) } WvProviderUnlockRemove(pProvider); } - -void WvDeviceOpen(WV_PROVIDER *pProvider, WDFREQUEST Request) -{ - WV_IO_ID *inid, *outid; - size_t inlen, outlen; - WV_DEVICE *dev; - NTSTATUS status; - ci_umv_buf_t verbsData; - - status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &inid, &inlen); - if (!NT_SUCCESS(status)) { - goto err1; - } - status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); - if (!NT_SUCCESS(status)) { - goto err1; - } - - dev = WvDeviceAlloc(pProvider); - if (dev == NULL) { - status = STATUS_NO_MEMORY; - goto err1; - } - - KeAcquireGuardedMutex(&pProvider->Lock); - WvProviderDisableRemove(pProvider); - KeReleaseGuardedMutex(&pProvider->Lock); - - WvInitVerbsData(&verbsData, inid->VerbInfo, inlen - sizeof(WV_IO_ID), - outlen - sizeof(WV_IO_ID), inid + 1); - status = WvDeviceInit(dev, inid->Id, &verbsData); - if (!NT_SUCCESS(status)) { - goto err2; - } - - KeAcquireGuardedMutex(&pProvider->Lock); - dev->Id = IndexListInsertHead(&pProvider->DevIndex, dev); - if (dev->Id == 0) { - status = STATUS_NO_MEMORY; - goto err2; - } - KeReleaseGuardedMutex(&pProvider->Lock); - - WvProviderEnableRemove(pProvider); - outid->Id = dev->Id; - outid->VerbInfo = verbsData.status; - WdfRequestCompleteWithInformation(Request, status, outlen); - return; - -err2: - WvDeviceFree(dev); - WvProviderEnableRemove(pProvider); -err1: - WdfRequestComplete(Request, status); -} - -void WvDeviceClose(WV_PROVIDER *pProvider, WDFREQUEST Request) -{ - WV_DEVICE *dev; - UINT64 *id; - NTSTATUS status; - - status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL); - if (!NT_SUCCESS(status)) { - goto out; - } - - KeAcquireGuardedMutex(&pProvider->Lock); - WvProviderDisableRemove(pProvider); - dev = IndexListAt(&pProvider->DevIndex, (SIZE_T) id); - if (dev == NULL) { - status = STATUS_NO_SUCH_DEVICE; - } else if (dev->Ref > 1) { - status = STATUS_ACCESS_DENIED; - } else { - IndexListRemove(&pProvider->DevIndex, (SIZE_T) id); - status = STATUS_SUCCESS; - } - KeReleaseGuardedMutex(&pProvider->Lock); - - if (NT_SUCCESS(status)) { - WvDeviceFree(dev); - } - WvProviderEnableRemove(pProvider); -out: - WdfRequestComplete(Request, status); -} diff --git a/branches/winverbs/core/winverbs/kernel/wv_provider.h b/branches/winverbs/core/winverbs/kernel/wv_provider.h index 00f720d1..efcea295 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_provider.h +++ b/branches/winverbs/core/winverbs/kernel/wv_provider.h @@ -47,7 +47,10 @@ typedef struct _WV_PROVIDER { LIST_ENTRY Entry; INDEX_LIST DevIndex; + INDEX_LIST CqIndex; INDEX_LIST PdIndex; + INDEX_LIST SrqIndex; + INDEX_LIST QpIndex; INDEX_LIST MwIndex; INDEX_LIST AhIndex; @@ -71,7 +74,4 @@ void WvProviderRemoveHandler(WV_PROVIDER *pProvider, WV_RDMA_DEVICE *pDevice); void WvProviderDisableRemove(WV_PROVIDER *pProvider); void WvProviderEnableRemove(WV_PROVIDER *pProvider); -void WvDeviceOpen(WV_PROVIDER *pProvider, WDFREQUEST Request); -void WvDeviceClose(WV_PROVIDER *pProvider, WDFREQUEST Request); - #endif // _WV_PROVIDER_H_ diff --git a/branches/winverbs/core/winverbs/kernel/wv_qp.c b/branches/winverbs/core/winverbs/kernel/wv_qp.c new file mode 100644 index 00000000..6b10b4b4 --- /dev/null +++ b/branches/winverbs/core/winverbs/kernel/wv_qp.c @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2008 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "wv_qp.h" +#include "wv_ioctl.h" + +static void WvVerbsConvertCreate(ib_qp_create_t *pVerbsAttr, + WV_IO_QP_CREATE *pAttr, WV_QUEUE_PAIR *pQp) +{ + pVerbsAttr->h_sq_cq = pQp->pSendCq->hVerbsCq; + pVerbsAttr->h_rq_cq = pQp->pReceiveCq->hVerbsCq; + pVerbsAttr->h_srq = (pQp->pSrq != NULL) ? pQp->pSrq->hVerbsSrq : NULL; + + pVerbsAttr->qp_type = (ib_qp_type_t) pAttr->QpType; + pVerbsAttr->sq_depth = pAttr->SendDepth; + pVerbsAttr->rq_depth = pAttr->ReceiveDepth; + pVerbsAttr->sq_sge = pAttr->SendSge; + pVerbsAttr->rq_sge = pAttr->ReceiveSge; + + pVerbsAttr->sq_signaled = pAttr->SignalSends; +} + +static UINT8 WvQpStateConvertVerbs(ib_qp_state_t State) +{ + switch (State) { + case IB_QPS_RESET: return WV_IO_QP_STATE_RESET; + case IB_QPS_INIT: return WV_IO_QP_STATE_INIT; + case IB_QPS_RTR: return WV_IO_QP_STATE_RTR; + case IB_QPS_RTS: return WV_IO_QP_STATE_RTS; + case IB_QPS_SQD: return WV_IO_QP_STATE_SDQ; + case IB_QPS_SQERR: return WV_IO_QP_STATE_SQERROR; + default: return WV_IO_QP_STATE_ERROR; + } +} + +static void WvAvConvertVerbs(WV_IO_AV *pAv, ib_av_attr_t *pVerbsAv) +{ + if (pVerbsAv->grh_valid) { + RtlCopyMemory(pAv, &pVerbsAv->grh, sizeof pVerbsAv->grh); + pAv->GrhValid = 1; + } else { + pAv->GrhValid = 0; + } + + pAv->PortNumber = pVerbsAv->port_num; + pAv->ServiceLevel = pVerbsAv->sl; + pAv->DLid = pVerbsAv->dlid; + pAv->StaticRate = pVerbsAv->static_rate; + pAv->SourcePathBits = pVerbsAv->path_bits; +} + +static void WvQpAttrConvertVerbs(WV_IO_QP_ATTRIBUTES *pAttr, ib_qp_attr_t *pVerbsAttr) +{ + pAttr->QpType = pVerbsAttr->qp_type; + pAttr->AccessFlags = pVerbsAttr->access_ctrl; + pAttr->PkeyIndex = pVerbsAttr->pkey_index; + + pAttr->MaxInlineSend = pVerbsAttr->sq_max_inline; + pAttr->SendDepth = pVerbsAttr->sq_depth; + pAttr->ReceiveDepth = pVerbsAttr->rq_depth; + pAttr->SendSge = pVerbsAttr->sq_sge; + pAttr->ReceiveSge = pVerbsAttr->rq_sge; + pAttr->InitiatorDepth = pVerbsAttr->init_depth; + pAttr->ResponderResources = pVerbsAttr->resp_res; + + pAttr->Options = 0; + pAttr->QpType = pVerbsAttr->qp_type; + pAttr->CurrentQpState = WvQpStateConvertVerbs(pVerbsAttr->state); + pAttr->QpState = pAttr->CurrentQpState; + pAttr->ApmState = pVerbsAttr->apm_state; + pAttr->Qpn = pVerbsAttr->num; + pAttr->DestinationQpn = pAttr->DestinationQpn; + pAttr->Qkey = pVerbsAttr->qkey; + pAttr->SendPsn = pVerbsAttr->sq_psn; + pAttr->ReceivePsn = pVerbsAttr->rq_psn; + + pAttr->PrimaryAddressVector.PortNumber = pVerbsAttr->primary_port; + pAttr->AlternateAddressVector.PortNumber = pVerbsAttr->alternate_port; + WvAvConvertVerbs(&pAttr->PrimaryAddressVector, &pVerbsAttr->primary_av); + WvAvConvertVerbs(&pAttr->AlternateAddressVector, &pVerbsAttr->alternate_av); + pAttr->SignalSends = (UINT16) pVerbsAttr->sq_signaled; + pAttr->AccessFlags = pVerbsAttr->access_ctrl; + pAttr->PkeyIndex = pVerbsAttr->pkey_index; + + pAttr->PathMtu = pVerbsAttr->primary_av.conn.path_mtu; + pAttr->LocalAckTimeout = pVerbsAttr->primary_av.conn.local_ack_timeout; + pAttr->SequenceErrorRetryCount = pVerbsAttr->primary_av.conn.seq_err_retry_cnt; + pAttr->RnrRetryCount = pVerbsAttr->primary_av.conn.rnr_retry_cnt; +} + +static void WvQpGet(WV_QUEUE_PAIR *pQp) +{ + InterlockedIncrement(&pQp->Ref); +} + +static void WvQpPut(WV_QUEUE_PAIR *pQp) +{ + if (InterlockedDecrement(&pQp->Ref) == 0) { + KeSetEvent(&pQp->Event, 0, FALSE); + } +} + +WV_QUEUE_PAIR *WvQpAcquire(WV_PROVIDER *pProvider, UINT64 Id) +{ + WV_QUEUE_PAIR *qp; + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + qp = IndexListAt(&pProvider->QpIndex, (SIZE_T) Id); + if (qp != NULL && qp->hVerbsQp != NULL) { + WvQpGet(qp); + } else { + qp = NULL; + WvProviderEnableRemove(pProvider); + } + KeReleaseGuardedMutex(&pProvider->Lock); + + return qp; +} + +void WvQpRelease(WV_QUEUE_PAIR *pQp) +{ + WvProviderEnableRemove(pQp->pProvider); + WvQpPut(pQp); +} + +static NTSTATUS WvQpCreateAcquire(WV_PROVIDER *pProvider, WV_QUEUE_PAIR *pQp, + WV_IO_QP_CREATE *pAttributes) +{ + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + pQp->pPd = IndexListAt(&pProvider->PdIndex, (SIZE_T) pAttributes->Id.Id); + if (pQp->pPd != NULL && pQp->pPd->hVerbsPd != NULL) { + WvPdGet(pQp->pPd); + pQp->pVerbs = pQp->pPd->pVerbs; + } else { + goto err1; + } + + pQp->pSendCq = IndexListAt(&pProvider->CqIndex, (SIZE_T) pAttributes->SendCqId); + if (pQp->pSendCq != NULL && pQp->pSendCq->hVerbsCq != NULL) { + WvCqGet(pQp->pSendCq); + } else { + goto err2; + } + + pQp->pReceiveCq = IndexListAt(&pProvider->CqIndex, + (SIZE_T) pAttributes->ReceiveCqId); + if (pQp->pReceiveCq != NULL && pQp->pReceiveCq->hVerbsCq != NULL) { + WvCqGet(pQp->pReceiveCq); + } else { + goto err3; + } + + if (pAttributes->SrqId != 0) { + pQp->pSrq = IndexListAt(&pProvider->SrqIndex, (SIZE_T) pAttributes->SrqId); + if (pQp->pSrq != NULL && pQp->pSrq->hVerbsSrq != NULL) { + WvSrqGet(pQp->pSrq); + } else { + goto err4; + } + } else { + pQp->pSrq = NULL; + } + KeReleaseGuardedMutex(&pProvider->Lock); + + return STATUS_SUCCESS; + +err4: + WvCqPut(pQp->pReceiveCq); +err3: + WvCqPut(pQp->pSendCq); +err2: + WvPdPut(pQp->pPd); +err1: + WvProviderEnableRemove(pProvider); + KeReleaseGuardedMutex(&pProvider->Lock); + return STATUS_NOT_FOUND; +} + +static void WvQpCreateRelease(WV_PROVIDER *pProvider, WV_QUEUE_PAIR *pQp) +{ + WvProviderEnableRemove(pProvider); + if (pQp->pSrq != NULL) { + WvSrqPut(pQp->pSrq); + } + WvCqPut(pQp->pReceiveCq); + WvCqPut(pQp->pSendCq); + WvPdPut(pQp->pPd); +} + +void WvQpCreate(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_QP_CREATE *attr; + WV_IO_ID *outid; + size_t inlen, outlen; + WV_QUEUE_PAIR *qp; + ib_qp_create_t create; + ib_qp_attr_t ib_attr; + NTSTATUS status; + ib_api_status_t ib_status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_QP_CREATE), + &attr, &inlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + + qp = ExAllocatePoolWithTag(PagedPool, sizeof(WV_QUEUE_PAIR), 'pqvw'); + if (qp == NULL) { + status = STATUS_NO_MEMORY; + goto err1; + } + + qp->Ref = 1; + KeInitializeEvent(&qp->Event, NotificationEvent, FALSE); + status = WvQpCreateAcquire(pProvider, qp, attr); + if (!NT_SUCCESS(status)) { + goto err2; + } + + WvVerbsConvertCreate(&create, attr, qp); + WvInitVerbsData(&verbsData, attr->Id.VerbInfo, inlen - sizeof(WV_IO_QP_CREATE), + outlen - sizeof(WV_IO_ID), attr + 1); + ib_status = qp->pVerbs->create_qp(qp->pPd->hVerbsPd, qp, &create, &ib_attr, + &qp->hVerbsQp, &verbsData); + if (ib_status != IB_SUCCESS) { + goto err3; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + outid->Id = IndexListInsertHead(&pProvider->QpIndex, qp); + if (outid->Id == 0) { + status = STATUS_NO_MEMORY; + goto err4; + } + InsertHeadList(&qp->pPd->QpList, &qp->Entry); + KeReleaseGuardedMutex(&pProvider->Lock); + + WvProviderEnableRemove(pProvider); + outid->VerbInfo = verbsData.status; + WdfRequestCompleteWithInformation(Request, status, outlen); + return; + +err4: + KeReleaseGuardedMutex(&pProvider->Lock); + qp->pVerbs->destroy_qp(qp->hVerbsQp, 0); +err3: + WvQpCreateRelease(pProvider, qp); +err2: + ExFreePool(qp); +err1: + WdfRequestComplete(Request, status); +} + +void WvQpDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_QUEUE_PAIR *qp; + UINT64 *id; + NTSTATUS status; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL); + if (!NT_SUCCESS(status)) { + goto out; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + qp = IndexListAt(&pProvider->QpIndex, (SIZE_T) *id); + if (qp == NULL) { + status = STATUS_NOT_FOUND; + } else if (qp->Ref > 1) { + status = STATUS_ACCESS_DENIED; + } else { + IndexListRemove(&pProvider->QpIndex, (SIZE_T) *id); + RemoveEntryList(&qp->Entry); + status = STATUS_SUCCESS; + } + KeReleaseGuardedMutex(&pProvider->Lock); + + if (NT_SUCCESS(status)) { + WvQpFree(qp); + } + WvProviderEnableRemove(pProvider); +out: + WdfRequestComplete(Request, status); +} + +void WvQpFree(WV_QUEUE_PAIR *pQp) +{ + if (InterlockedDecrement(&pQp->Ref) > 0) { + KeWaitForSingleObject(&pQp->Event, Executive, KernelMode, FALSE, NULL); + } + + if (pQp->hVerbsQp != NULL) { + pQp->pVerbs->destroy_qp(pQp->hVerbsQp, 0); + } + + if (pQp->pSrq != NULL) { + WvSrqPut(pQp->pSrq); + } + WvCqPut(pQp->pReceiveCq); + WvCqPut(pQp->pSendCq); + WvPdPut(pQp->pPd); + ExFreePool(pQp); +} + +void WvQpQuery(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_QP_ATTRIBUTES *pattr; + WV_IO_ID *id; + size_t inlen, outlen, len = 0; + WV_QUEUE_PAIR *qp; + ib_qp_attr_t attr; + NTSTATUS status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &id, &inlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_QP_ATTRIBUTES), + &pattr, &outlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + + qp = WvQpAcquire(pProvider, id->Id); + if (qp == NULL) { + status = STATUS_NOT_FOUND; + goto complete; + } + + WvInitVerbsData(&verbsData, id->VerbInfo, inlen - sizeof(WV_IO_ID), + outlen - sizeof(WV_IO_QP_ATTRIBUTES), id + 1); + qp->pVerbs->query_qp(qp->hVerbsQp, &attr, &verbsData); + WvQpRelease(qp); + + WvQpAttrConvertVerbs(pattr, &attr); + pattr->Id.VerbInfo = verbsData.status; + len = outlen; + +complete: + WdfRequestCompleteWithInformation(Request, status, len); +} diff --git a/branches/winverbs/core/winverbs/kernel/wv_qp.h b/branches/winverbs/core/winverbs/kernel/wv_qp.h index e0b840b7..fb726ec5 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_qp.h +++ b/branches/winverbs/core/winverbs/kernel/wv_qp.h @@ -32,24 +32,42 @@ #ifndef _WV_QP_H_ #define _WV_QP_H_ +#include #include #include #include + #include "wv_pd.h" +#include "wv_provider.h" #include "wv_cq.h" #include "wv_srq.h" typedef struct _WV_QUEUE_PAIR { + WV_PROVIDER *pProvider; WV_PROTECTION_DOMAIN *pPd; WV_COMPLETION_QUEUE *pSendCq; WV_COMPLETION_QUEUE *pReceiveCq; WV_SHARED_RECEIVE_QUEUE *pSrq; ci_interface_t *pVerbs; - ib_qp_handle_t m_hVerbsQp; - UINT64 m_Id; - LONG nRef; + ib_qp_handle_t hVerbsQp; + LIST_ENTRY Entry; + + KEVENT Event; + LONG Ref; } WV_QUEUE_PAIR; +void WvQpCreate(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvQpDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvQpFree(WV_QUEUE_PAIR *pQp); + +//void WvQpGet(WV_QUEUE_PAIR *pQp); +//void WvQpPut(WV_QUEUE_PAIR *pQp); + +//void WvQpModify(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvQpQuery(WV_PROVIDER *pProvider, WDFREQUEST Request); +//void WvQpAttach(WV_PROVIDER *pProvider, WDFREQUEST Request); +//void WvQpDetach(WV_PROVIDER *pProvider, WDFREQUEST Request); + #endif // _WV_QP_H_ diff --git a/branches/winverbs/core/winverbs/kernel/wv_srq.c b/branches/winverbs/core/winverbs/kernel/wv_srq.c new file mode 100644 index 00000000..2c3c5b4f --- /dev/null +++ b/branches/winverbs/core/winverbs/kernel/wv_srq.c @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2008 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "wv_srq.h" +#include "wv_ioctl.h" + +void WvSrqGet(WV_SHARED_RECEIVE_QUEUE *pSrq) +{ + InterlockedIncrement(&pSrq->Ref); +} + +void WvSrqPut(WV_SHARED_RECEIVE_QUEUE *pSrq) +{ + if (InterlockedDecrement(&pSrq->Ref) == 0) { + KeSetEvent(&pSrq->Event, 0, FALSE); + } +} + +WV_SHARED_RECEIVE_QUEUE *WvSrqAcquire(WV_PROVIDER *pProvider, UINT64 Id) +{ + WV_SHARED_RECEIVE_QUEUE *srq; + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + srq = IndexListAt(&pProvider->SrqIndex, (SIZE_T) Id); + if (srq != NULL && srq->hVerbsSrq != NULL) { + WvSrqGet(srq); + } else { + srq = NULL; + WvProviderEnableRemove(pProvider); + } + KeReleaseGuardedMutex(&pProvider->Lock); + + return srq; +} + +void WvSrqRelease(WV_SHARED_RECEIVE_QUEUE *pSrq) +{ + WvProviderEnableRemove(pSrq->pProvider); + WvSrqPut(pSrq); +} + +static void WvVerbsConvertSrq(ib_srq_attr_t *pVerbsAttr, + WV_IO_SRQ_ATTRIBUTES *pAttr) +{ + pVerbsAttr->max_wr = pAttr->MaxWr; + pVerbsAttr->max_sge = pAttr->MaxSge; + pVerbsAttr->srq_limit = pAttr->SrqLimit; +} + +static NTSTATUS WvSrqAlloc(WV_PROTECTION_DOMAIN *pPd, WV_IO_SRQ_ATTRIBUTES *pAttr, + WV_SHARED_RECEIVE_QUEUE **ppSrq, ci_umv_buf_t *pVerbsData) +{ + WV_SHARED_RECEIVE_QUEUE *srq; + ib_srq_attr_t attr; + ib_api_status_t ib_status; + + srq = ExAllocatePoolWithTag(PagedPool, sizeof(WV_SHARED_RECEIVE_QUEUE), 'rsvw'); + if (srq == NULL) { + return STATUS_NO_MEMORY; + } + + srq->Ref = 1; + KeInitializeEvent(&srq->Event, NotificationEvent, FALSE); + + WvVerbsConvertSrq(&attr, pAttr); + ib_status = pPd->pVerbs->create_srq(pPd->hVerbsPd, srq, &attr, + &srq->hVerbsSrq, pVerbsData); + if (ib_status != IB_SUCCESS) { + goto err; + } + + WvPdGet(pPd); + srq->pPd = pPd; + srq->pProvider = pPd->pDevice->pProvider; + srq->pVerbs = pPd->pVerbs; + *ppSrq = srq; + return STATUS_SUCCESS; + +err: + ExFreePool(srq); + return STATUS_UNSUCCESSFUL; +} + +void WvSrqCreate(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_SRQ_ATTRIBUTES *attr; + WV_IO_ID *outid; + size_t inlen, outlen; + WV_PROTECTION_DOMAIN *pd; + WV_SHARED_RECEIVE_QUEUE *srq; + NTSTATUS status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_SRQ_ATTRIBUTES), + &attr, &inlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_ID), &outid, &outlen); + if (!NT_SUCCESS(status)) { + goto err1; + } + + pd = WvPdAcquire(pProvider, attr->Id.Id); + if (pd == NULL) { + status = STATUS_NOT_FOUND; + goto err1; + } + + WvInitVerbsData(&verbsData, attr->Id.VerbInfo, inlen - sizeof(WV_IO_SRQ_ATTRIBUTES), + outlen - sizeof(WV_IO_ID), attr + 1); + status = WvSrqAlloc(pd, attr, &srq, &verbsData); + if (!NT_SUCCESS(status)) { + goto err2; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + outid->Id = IndexListInsertHead(&pProvider->SrqIndex, srq); + if (outid->Id == 0) { + status = STATUS_NO_MEMORY; + goto err3; + } + InsertHeadList(&pd->SrqList, &srq->Entry); + KeReleaseGuardedMutex(&pProvider->Lock); + + WvPdRelease(pd); + outid->VerbInfo = verbsData.status; + WdfRequestCompleteWithInformation(Request, status, outlen); + return; + +err3: + KeReleaseGuardedMutex(&pProvider->Lock); + WvSrqFree(srq); +err2: + WvPdRelease(pd); +err1: + WdfRequestComplete(Request, status); +} + +void WvSrqDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_SHARED_RECEIVE_QUEUE *srq; + UINT64 *id; + NTSTATUS status; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL); + if (!NT_SUCCESS(status)) { + goto out; + } + + KeAcquireGuardedMutex(&pProvider->Lock); + WvProviderDisableRemove(pProvider); + srq = IndexListAt(&pProvider->SrqIndex, (SIZE_T) *id); + if (srq == NULL) { + status = STATUS_NOT_FOUND; + } else if (srq->Ref > 1) { + status = STATUS_ACCESS_DENIED; + } else { + IndexListRemove(&pProvider->SrqIndex, (SIZE_T) *id); + RemoveEntryList(&srq->Entry); + status = STATUS_SUCCESS; + } + KeReleaseGuardedMutex(&pProvider->Lock); + + if (NT_SUCCESS(status)) { + WvSrqFree(srq); + } + WvProviderEnableRemove(pProvider); +out: + WdfRequestComplete(Request, status); +} + +void WvSrqFree(WV_SHARED_RECEIVE_QUEUE *pSrq) +{ + if (InterlockedDecrement(&pSrq->Ref) > 0) { + KeWaitForSingleObject(&pSrq->Event, Executive, KernelMode, FALSE, NULL); + } + + if (pSrq->hVerbsSrq != NULL) { + pSrq->pVerbs->destroy_srq(pSrq->hVerbsSrq); + } + + WvPdPut(pSrq->pPd); + ExFreePool(pSrq); +} + +void WvSrqModify(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_SRQ_ATTRIBUTES *pattr; + UINT32 *verbInfo; + size_t inlen, outlen, len = 0; + WV_SHARED_RECEIVE_QUEUE *srq; + ib_srq_attr_t attr; + NTSTATUS status; + ib_api_status_t ib_status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_SRQ_ATTRIBUTES), + &pattr, &inlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(UINT32), &verbInfo, &outlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + + srq = WvSrqAcquire(pProvider, pattr->Id.Id); + if (srq == NULL) { + status = STATUS_NOT_FOUND; + goto complete; + } + + WvInitVerbsData(&verbsData, pattr->Id.VerbInfo, inlen - sizeof(WV_IO_SRQ_ATTRIBUTES), + outlen - sizeof(UINT32), pattr + 1); + WvVerbsConvertSrq(&attr, pattr); + ib_status = srq->pVerbs->modify_srq(srq->hVerbsSrq, &attr, + IB_SRQ_MAX_WR | IB_SRQ_LIMIT, &verbsData); + WvSrqRelease(srq); + + if (ib_status != IB_SUCCESS) { + status = STATUS_UNSUCCESSFUL; + } + + len = outlen; + *verbInfo = verbsData.status; +complete: + WdfRequestCompleteWithInformation(Request, status, len); +} + +void WvSrqQuery(WV_PROVIDER *pProvider, WDFREQUEST Request) +{ + WV_IO_SRQ_ATTRIBUTES *pattr; + WV_IO_ID *id; + size_t inlen, outlen, len = 0; + WV_SHARED_RECEIVE_QUEUE *srq; + ib_srq_attr_t attr; + NTSTATUS status; + ci_umv_buf_t verbsData; + + status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &id, &inlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_SRQ_ATTRIBUTES), + &pattr, &outlen); + if (!NT_SUCCESS(status)) { + goto complete; + } + + srq = WvSrqAcquire(pProvider, id->Id); + if (srq == NULL) { + status = STATUS_NOT_FOUND; + goto complete; + } + + WvInitVerbsData(&verbsData, id->VerbInfo, inlen - sizeof(WV_IO_ID), + outlen - sizeof(WV_IO_SRQ_ATTRIBUTES), id + 1); + srq->pVerbs->query_srq(srq->hVerbsSrq, &attr, &verbsData); + WvSrqRelease(srq); + + pattr->Id.VerbInfo = verbsData.status; + pattr->MaxWr = attr.max_wr; + pattr->MaxSge = attr.max_sge; + pattr->SrqLimit = attr.srq_limit; + len = outlen; +complete: + WdfRequestCompleteWithInformation(Request, status, len); +} \ No newline at end of file diff --git a/branches/winverbs/core/winverbs/kernel/wv_srq.h b/branches/winverbs/core/winverbs/kernel/wv_srq.h index 7a509d7b..1516cb5f 100644 --- a/branches/winverbs/core/winverbs/kernel/wv_srq.h +++ b/branches/winverbs/core/winverbs/kernel/wv_srq.h @@ -32,19 +32,37 @@ #ifndef _WV_SRQ_H_ #define _WV_SRQ_H_ +#include #include #include #include + #include "wv_pd.h" +#include "wv_provider.h" typedef struct _WV_SHARED_RECEIVE_QUEUE { + WV_PROVIDER *pProvider; WV_PROTECTION_DOMAIN *pPd; ci_interface_t *pVerbs; ib_srq_handle_t hVerbsSrq; - UINT64 Id; - LONG nRef; + LIST_ENTRY Entry; + + KEVENT Event; + LONG Ref; } WV_SHARED_RECEIVE_QUEUE; +void WvSrqCreate(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvSrqDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvSrqFree(WV_SHARED_RECEIVE_QUEUE *pSrq); + +WV_SHARED_RECEIVE_QUEUE *WvSrqAcquire(WV_PROVIDER *pProvider, UINT64 Id); +void WvSrqRelease(WV_SHARED_RECEIVE_QUEUE *pSrq); +void WvSrqGet(WV_SHARED_RECEIVE_QUEUE *pSrq); +void WvSrqPut(WV_SHARED_RECEIVE_QUEUE *pSrq); + +void WvSrqModify(WV_PROVIDER *pProvider, WDFREQUEST Request); +void WvSrqQuery(WV_PROVIDER *pProvider, WDFREQUEST Request); + #endif // _WV_SRQ_H_ -- 2.41.0