winverbs/lib: cache pkey table

Connection establishment rates take a huge performance hit
as a result of using FindPkey, which results in querying the
HCA to lookup current pkey information.  Add a cache to the
userspace library for pkey information.

The cache is updated using the CWVDevice:Notify() routine.
Before any access is made to the cache, the caller checks
for changes to the partition tables.  If a change is found,
then the cache is updated.  Otherwise, the cache is accessed
to map pkey values to indices.

This results in the connection rate over winverbs almost
doubling.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1@2672 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/core/winverbs/user/wv_device.cpp b/trunk/core/winverbs/user/wv_device.cpp
index 9aac78895b942283fcd647f451b8d31c890b1f52..b260d418b2cdd97dbc4928386a8df9ac13ec0d9a 100644 (file)
--- a/trunk/core/winverbs/user/wv_device.cpp
+++ b/trunk/core/winverbs/user/wv_device.cpp
@@ -48,6 +48,8 @@ CWVDevice::CWVDevice(CWVProvider *pProvider)
        m_pProvider = pProvider;\r
        m_hFile = pProvider->m_hFile;\r
 \r
+       m_PortCount = 0;\r
+       m_pPorts = NULL;\r
        m_hVerbsDevice = NULL;\r
        m_hLib = NULL;\r
 }\r
@@ -121,11 +123,56 @@ post:
                hr = WvConvertIbStatus(stat);\r
        }\r
 \r
+       if (SUCCEEDED(hr)) {\r
+               hr = InitPorts();\r
+       }\r
+\r
+       return hr;\r
+}\r
+\r
+STDMETHODIMP CWVDevice::\r
+InitPorts()\r
+{\r
+       WV_DEVICE_ATTRIBUTES attr;\r
+       WV_PORT *port;\r
+       HRESULT hr;\r
+\r
+       hr = Query(&attr);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       m_pPorts = new WV_PORT[attr.PhysPortCount];\r
+       if (m_pPorts == NULL) {\r
+               return WV_NO_MEMORY;\r
+       }\r
+\r
+       RtlZeroMemory(m_pPorts, sizeof(WV_PORT) * attr.PhysPortCount);\r
+       for (m_PortCount = 0; m_PortCount < attr.PhysPortCount; m_PortCount++) {\r
+               port = &m_pPorts[m_PortCount];\r
+\r
+               port->m_Overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);\r
+               if (port->m_Overlap.hEvent == NULL) {\r
+                       hr = WV_INSUFFICIENT_RESOURCES;\r
+                       break;\r
+               }\r
+\r
+               port->m_Overlap.hEvent = (HANDLE) ((ULONG_PTR) port->m_Overlap.hEvent | 1);\r
+\r
+               port->m_Flags = 0xFFFFFFFF;\r
+               hr = UpdatePort(m_PortCount + 1);\r
+               if (FAILED(hr)) {\r
+                       CloseHandle(port->m_Overlap.hEvent);\r
+                       break;\r
+               }\r
+       }\r
+\r
        return hr;\r
 }\r
 \r
 CWVDevice::~CWVDevice()\r
 {\r
+       WV_PORT *port;\r
        DWORD   bytes;\r
        HRESULT hr;\r
 \r
@@ -137,6 +184,16 @@ CWVDevice::~CWVDevice()
                m_Verbs.post_close_ca(m_hVerbsDevice, (ib_api_status_t) hr);\r
        }\r
 \r
+       while (m_PortCount--) {\r
+               port = &m_pPorts[m_PortCount];\r
+               GetOverlappedResult(&port->m_Overlap, &bytes, TRUE);\r
+               CloseHandle(port->m_Overlap.hEvent);\r
+       }\r
+\r
+       if (m_pPorts != NULL) {\r
+               delete m_pPorts;\r
+       }\r
+\r
        if (m_hLib != NULL) {\r
                FreeLibrary(m_hLib);\r
        }\r
@@ -252,6 +309,26 @@ QueryPort(UINT8 PortNumber, WV_PORT_ATTRIBUTES* pAttributes)
                return HRESULT_FROM_WIN32(GetLastError());\r
        }\r
 \r
+       pAttributes->PkeyTableLength = min(pAttributes->PkeyTableLength, WV_MAX_PKEYS);\r
+       return WV_SUCCESS;\r
+}\r
+\r
+STDMETHODIMP CWVDevice::\r
+UpdatePort(UINT8 PortNumber)\r
+{\r
+       WV_PORT *port = &m_pPorts[PortNumber - 1];\r
+       HRESULT hr;\r
+\r
+       if (port->m_Flags & WV_EVENT_PARTITION) {\r
+               UpdatePkeys(PortNumber);\r
+       }\r
+\r
+       port->m_Flags = 0;\r
+       hr = Notify(PortNumber, &port->m_Overlap, &port->m_Flags);\r
+       if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               return hr;\r
+       }\r
+\r
        return WV_SUCCESS;\r
 }\r
 \r
@@ -316,63 +393,73 @@ FindGid(UINT8 PortNumber, WV_GID *pGid, DWORD *pIndex)
 }\r
 \r
 STDMETHODIMP CWVDevice::\r
-QueryPkey(UINT8 PortNumber, UINT16 Index, NET16* pPkey)\r
+UpdatePkeys(UINT8 PortNumber)\r
 {\r
        WV_IO_DEVICE_PORT_QUERY query;\r
-       NET16                                   *pkeytable;\r
-       DWORD                                   npkey, bytes;\r
-       HRESULT                                 hr;\r
-       CWVBuffer                               buf;\r
+       WV_PORT *port;\r
+       DWORD   bytes;\r
 \r
-       bytes = sizeof NET16 * (Index + 1);\r
-       pkeytable = (NET16 *) buf.Get(bytes);\r
-       if (pkeytable == NULL) {\r
-               return WV_NO_MEMORY;\r
-       }\r
+       port = &m_pPorts[PortNumber - 1];\r
+       bytes = sizeof(port->m_PkeyTable);\r
 \r
        query.Id = m_Id;\r
        query.PortNumber = PortNumber;\r
        RtlZeroMemory(&query.Reserved, sizeof query.Reserved);\r
 \r
        if (!WvDeviceIoControl(m_hFile, WV_IOCTL_DEVICE_PKEY_QUERY, &query,\r
-                                                  sizeof query, pkeytable, bytes, &bytes, NULL)) {\r
-               hr = HRESULT_FROM_WIN32(GetLastError());\r
-               goto out;\r
+                                                  sizeof query, port->m_PkeyTable, bytes, &bytes, NULL)) {\r
+               port->m_PkeyCount = 0;\r
+               return HRESULT_FROM_WIN32(GetLastError());\r
+       }\r
+\r
+       port->m_PkeyCount = (UINT16) (bytes / sizeof NET16);\r
+       return WV_SUCCESS;\r
+}\r
+\r
+STDMETHODIMP CWVDevice::\r
+QueryPkey(UINT8 PortNumber, UINT16 Index, NET16* pPkey)\r
+{\r
+       WV_PORT *port = &m_pPorts[PortNumber - 1];\r
+       HRESULT hr;\r
+\r
+       EnterCriticalSection(&m_CritSec);\r
+       if (HasOverlappedIoCompleted(&port->m_Overlap)) {\r
+               UpdatePort(PortNumber);\r
        }\r
 \r
-       npkey = bytes / sizeof NET16;\r
-       if (Index >= npkey) {\r
+       if (Index < port->m_PkeyCount) {\r
+               *pPkey = port->m_PkeyTable[Index];\r
+               hr = WV_SUCCESS;\r
+       } else {\r
                hr = WV_INVALID_PARAMETER_2;\r
-               goto out;\r
        }\r
-       *pPkey = pkeytable[Index];\r
-       hr = WV_SUCCESS;\r
 \r
-out:\r
-       buf.Put();\r
+       LeaveCriticalSection(&m_CritSec);\r
        return hr;\r
 }\r
 \r
 STDMETHODIMP CWVDevice::\r
 FindPkey(UINT8 PortNumber, NET16 Pkey, UINT16 *pIndex)\r
 {\r
-       NET16   key;\r
+       WV_PORT *port = &m_pPorts[PortNumber - 1];\r
        UINT16  index;\r
-       HRESULT hr;\r
+       HRESULT hr = WV_INVALID_ADDRESS;\r
 \r
-       for (index = 0; true; index++) {\r
-               hr = QueryPkey(PortNumber, index, &key);\r
-               if (FAILED(hr)) {\r
-                       return hr;\r
-               }\r
+       EnterCriticalSection(&m_CritSec);\r
+       if (HasOverlappedIoCompleted(&port->m_Overlap)) {\r
+               UpdatePort(PortNumber);\r
+       }\r
 \r
-               if (Pkey == key) {\r
-                       *pIndex = (UINT16) index;\r
-                       return WV_SUCCESS;\r
+       for (index = 0; index < port->m_PkeyCount; index++) {\r
+               if (Pkey == port->m_PkeyTable[index]) {\r
+                       *pIndex = index;\r
+                       hr = WV_SUCCESS;\r
+                       break;\r
                }\r
        }\r
 \r
-       return WV_INVALID_ADDRESS;\r
+       LeaveCriticalSection(&m_CritSec);\r
+       return hr;\r
 }\r
 \r
 STDMETHODIMP CWVDevice::\r

diff --git a/trunk/core/winverbs/user/wv_device.h b/trunk/core/winverbs/user/wv_device.h
index 7e98da1bb99a4739e38d5d068eab5a33551e9f23..9f4009cd2e02fd8db38a9cb1ba71d595d6135643 100644 (file)
--- a/trunk/core/winverbs/user/wv_device.h
+++ b/trunk/core/winverbs/user/wv_device.h
@@ -38,6 +38,18 @@
 #include "wv_provider.h"\r
 #include "wv_base.h"\r
 \r
+#define WV_MAX_PKEYS 16\r
+\r
+typedef struct _WV_PORT\r
+{\r
+       OVERLAPPED      m_Overlap;\r
+       DWORD           m_Flags;\r
+       UINT16          m_PkeyCount;\r
+       NET16           m_PkeyTable[WV_MAX_PKEYS];\r
+\r
+}      WV_PORT;\r
+\r
+\r
 class CWVDevice : IWVDevice, public CWVBase\r
 {\r
 public:\r
@@ -105,7 +117,13 @@ public:
 \r
 protected:\r
        HMODULE                 m_hLib;\r
-       STDMETHODIMP Open(NET64 Guid);\r
+       STDMETHODIMP    Open(NET64 Guid);\r
+\r
+       WV_PORT                 *m_pPorts;\r
+       UINT8                   m_PortCount;\r
+       STDMETHODIMP    InitPorts();\r
+       STDMETHODIMP    UpdatePort(UINT8 PortNumber);\r
+       STDMETHODIMP    UpdatePkeys(UINT8 PortNumber);\r
 };\r
 \r
 #endif // __WV_DEVICE_H_\r