[WSD] Add support for multiple CQs to allow more than 3000 sockets per process.

Signed-off-by: Fab Tillier (ftillier@silverstorm.com)
git-svn-id: svn://openib.tc.cornell.edu/gen1@91 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/ulp/wsd/user/ibsp_iblow.c b/trunk/ulp/wsd/user/ibsp_iblow.c
index 130d45d346d65d6da6246e83cf355418c29c450d..a01c3af2233377b652febdb97f3f9df592576250 100644 (file)
--- a/trunk/ulp/wsd/user/ibsp_iblow.c
+++ b/trunk/ulp/wsd/user/ibsp_iblow.c
@@ -32,9 +32,6 @@
 #include "ibspdll.h"\r
 \r
 \r
-static void ib_destroy_cq_tinfo( struct cq_thread_info *cq_tinfo );\r
-\r
-\r
 typedef struct _io_comp_info\r
 {\r
        struct ibsp_socket_info *p_socket;\r
@@ -455,13 +452,13 @@ ib_cq_thread(
                                         __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
                }\r
 \r
-       } while( (cq_tinfo->ib_cq_thread_exit_wanted != TRUE) ||\r
-                       cl_qlist_count( &cq_tinfo->done_wr_list ) );\r
+       } while( !cq_tinfo->ib_cq_thread_exit_wanted );\r
 \r
        cl_status = cl_waitobj_destroy( cq_tinfo->cq_waitobj );\r
        if( cl_status != CL_SUCCESS )\r
        {\r
-               IBSP_ERROR( ("cl_waitobj_destroy() (%d)\n", cl_status) );\r
+               IBSP_ERROR(\r
+                       ("cl_waitobj_destroy() returned %s\n", CL_STATUS_MSG(cl_status)) );\r
        }\r
        HeapFree( g_ibsp.heap, 0, cq_tinfo );\r
 \r
@@ -471,6 +468,7 @@ ib_cq_thread(
 }\r
 \r
 \r
+/* Called with the HCA's CQ lock held. */\r
 static struct cq_thread_info *\r
 ib_alloc_cq_tinfo(\r
                                        struct ibsp_hca                         *hca )\r
@@ -479,26 +477,26 @@ ib_alloc_cq_tinfo(
        ib_cq_create_t cq_create;\r
        ib_api_status_t status;\r
        cl_status_t cl_status;\r
-       int error;\r
 \r
        IBSP_ENTER( IBSP_DBG_HW );\r
 \r
-       cq_tinfo = HeapAlloc( g_ibsp.heap, HEAP_ZERO_MEMORY, sizeof(struct cq_thread_info) );\r
+       cq_tinfo = HeapAlloc(\r
+               g_ibsp.heap, HEAP_ZERO_MEMORY, sizeof(struct cq_thread_info) );\r
 \r
-       if( cq_tinfo == NULL )\r
+       if( !cq_tinfo )\r
        {\r
-               IBSP_ERROR( ("HeapAlloc() Failed.\n") );\r
-               error = TRUE;\r
-               goto done;\r
+               IBSP_ERROR_EXIT( ("HeapAlloc() Failed.\n") );\r
+               return NULL;\r
        }\r
 \r
        cl_status = cl_waitobj_create( FALSE, &cq_tinfo->cq_waitobj );\r
        if( cl_status != CL_SUCCESS )\r
        {\r
                cq_tinfo->cq_waitobj = NULL;\r
-               IBSP_ERROR( ("cl_waitobj_create() (%d)\n", cl_status) );\r
-               error = TRUE;\r
-               goto done;\r
+               ib_destroy_cq_tinfo( cq_tinfo );\r
+               IBSP_ERROR_EXIT(\r
+                       ("cl_waitobj_create() returned %s\n", CL_STATUS_MSG(cl_status)) );\r
+               return NULL;\r
        }\r
 \r
        cq_tinfo->hca = hca;\r
@@ -509,9 +507,9 @@ ib_alloc_cq_tinfo(
 \r
        if( cq_tinfo->ib_cq_thread == NULL )\r
        {\r
-               IBSP_ERROR( ("CreateThread failed.") );\r
-               error = TRUE;\r
-               goto done;\r
+               ib_destroy_cq_tinfo( cq_tinfo );\r
+               IBSP_ERROR_EXIT( ("CreateThread failed (%d)", GetLastError()) );\r
+               return NULL;\r
        }\r
 \r
        STAT_INC( thread_num );\r
@@ -527,9 +525,10 @@ ib_alloc_cq_tinfo(
                &cq_tinfo->cq );\r
        if( status )\r
        {\r
-               IBSP_ERROR( ("ib_create_cq failed (%d)\n", status) );\r
-               error = TRUE;\r
-               goto done;\r
+               ib_destroy_cq_tinfo( cq_tinfo );\r
+               IBSP_ERROR_EXIT(\r
+                       ("ib_create_cq returned %s\n", ib_get_err_str( status )) );\r
+               return NULL;\r
        }\r
 \r
        STAT_INC( cq_num );\r
@@ -537,34 +536,35 @@ ib_alloc_cq_tinfo(
        status = ib_rearm_cq( cq_tinfo->cq, FALSE );\r
        if( status )\r
        {\r
-               IBSP_ERROR( ("ib_rearm_cq failed (%d)\n", status) );\r
-               error = TRUE;\r
-               goto done;\r
+               ib_destroy_cq_tinfo( cq_tinfo );\r
+               IBSP_ERROR_EXIT(\r
+                       ("ib_rearm_cq returned %s\n", ib_get_err_str( status )) );\r
+               return NULL;\r
        }\r
 \r
-       cl_spinlock_init( &cq_tinfo->wr_mutex );\r
-       cl_qlist_init( &cq_tinfo->done_wr_list );\r
        cq_tinfo->cqe_size = IB_CQ_SIZE;\r
 \r
-       /* Only one CQ per HCA now */\r
-       hca->cq_tinfo = cq_tinfo;\r
-\r
-       error = FALSE;\r
-\r
-done:\r
-       if( error == TRUE )\r
+       if( hca->cq_tinfo )\r
        {\r
-               ib_destroy_cq_tinfo( cq_tinfo );\r
-               cq_tinfo = NULL;\r
+               __cl_primitive_insert(\r
+                       &hca->cq_tinfo->list_item, &cq_tinfo->list_item );\r
+       }\r
+       else\r
+       {\r
+               /* Setup the list entry to point to itself. */\r
+               cq_tinfo->list_item.p_next = &cq_tinfo->list_item;\r
+               cq_tinfo->list_item.p_prev = &cq_tinfo->list_item;\r
        }\r
 \r
-       IBSP_EXIT( IBSP_DBG_HW );\r
+       /* Upon allocation, the new CQ becomes the primary. */\r
+       hca->cq_tinfo = cq_tinfo;\r
 \r
+       IBSP_EXIT( IBSP_DBG_HW );\r
        return (cq_tinfo);\r
 }\r
 \r
 \r
-static void\r
+void\r
 ib_destroy_cq_tinfo(\r
                                        struct cq_thread_info           *cq_tinfo )\r
 {\r
@@ -576,29 +576,22 @@ ib_destroy_cq_tinfo(
 \r
        IBSP_ENTER( IBSP_DBG_HW );\r
 \r
-       if( cq_tinfo == NULL )\r
-       {\r
-               return;\r
-       }\r
+       CL_ASSERT( cq_tinfo );\r
+       CL_ASSERT( cq_tinfo->qp_count == 0 );\r
 \r
        if( cq_tinfo->cq )\r
        {\r
                wclist.p_next = NULL;\r
                free_wclist = &wclist;\r
 \r
-               while( ib_poll_cq( cq_tinfo->cq, &free_wclist, &done_wclist ) == IB_SUCCESS )\r
+               while( ib_poll_cq(\r
+                       cq_tinfo->cq, &free_wclist, &done_wclist ) == IB_SUCCESS )\r
                {\r
-                       IBSP_TRACE( IBSP_DBG_WQ, ("%s():%d:0x%x:0x%x: free=%p, done=%p\n",\r
-                               __FUNCTION__,\r
-                               __LINE__, GetCurrentProcessId(),\r
-                               GetCurrentThreadId(),\r
-                               free_wclist, done_wclist) );\r
+                       IBSP_TRACE1( IBSP_DBG_WQ,\r
+                               ("free=%p, done=%p\n", free_wclist, done_wclist) );\r
                }\r
 \r
-               IBSP_TRACE( IBSP_DBG_WQ, ("%s():%d:0x%x:0x%x: ib_destroy_cq() start..\n",\r
-                       __FUNCTION__,\r
-                       __LINE__, GetCurrentProcessId(),\r
-                       GetCurrentThreadId()) );\r
+               IBSP_TRACE4( IBSP_DBG_WQ, ("ib_destroy_cq() start..\n") );\r
 \r
                /*\r
                 * Called from cleanup thread, okay to block.\r
@@ -606,13 +599,12 @@ ib_destroy_cq_tinfo(
                status = ib_destroy_cq( cq_tinfo->cq, ib_sync_destroy );\r
                if( status )\r
                {\r
-                       IBSP_ERROR( ("ib_destroy_cq failed (%d)\n", status) );\r
+                       IBSP_ERROR(\r
+                               ("ib_destroy_cq returned %s\n", ib_get_err_str( status )) );\r
                }\r
                else\r
                {\r
-                       IBSP_TRACE( IBSP_DBG_WQ,\r
-                               ("%s():%d:0x%x:0x%x: ib_destroy_cq() finished.\n", __FUNCTION__,\r
-                               __LINE__, GetCurrentProcessId(), GetCurrentThreadId()) );\r
+                       IBSP_TRACE4( IBSP_DBG_WQ, ("ib_destroy_cq() finished.\n") );\r
 \r
                        cq_tinfo->cq = NULL;\r
 \r
@@ -620,9 +612,6 @@ ib_destroy_cq_tinfo(
                }\r
        }\r
 \r
-       /* Currently only 1 CQ per HCA */\r
-       cq_tinfo->hca = NULL;\r
-\r
        if( cq_tinfo->ib_cq_thread )\r
        {\r
                /* ib_cq_thread() will release the cq_tinfo before exit. Don't\r
@@ -676,21 +665,21 @@ static struct cq_thread_info *
 ib_acquire_cq_tinfo(\r
                                        struct ibsp_hca                         *hca )\r
 {\r
-       struct cq_thread_info *cq_tinfo = NULL;\r
-       uint32_t current_cqe_size;\r
+       struct cq_thread_info   *cq_tinfo = NULL;\r
+       uint32_t                                cqe_size;\r
+       ib_api_status_t                 status;\r
 \r
        IBSP_ENTER( IBSP_DBG_HW );\r
 \r
-       /* \r
-        * TODO: If future implementations require more than 1 cq_tinfo per HCA, then\r
-        *   search HCA cq_tinfo list for optimal cq_tinfo \r
-        */\r
-       if( hca->cq_tinfo == NULL )\r
+       cl_spinlock_acquire( &hca->cq_lock );\r
+\r
+       if( !hca->cq_tinfo )\r
        {\r
                cq_tinfo = ib_alloc_cq_tinfo( hca );\r
-               if( cq_tinfo == NULL )\r
+               if( !cq_tinfo )\r
                {\r
-                       IBSP_ERROR( ("ib_alloc_cq_tinfo() failed\n") );\r
+                       IBSP_ERROR_EXIT( ("ib_alloc_cq_tinfo() failed\n") );\r
+                       cl_spinlock_release( &hca->cq_lock );\r
                        return (NULL);\r
                }\r
        }\r
@@ -701,38 +690,45 @@ ib_acquire_cq_tinfo(
 \r
        CL_ASSERT( cq_tinfo != NULL );\r
 \r
-       current_cqe_size = cq_tinfo->qp_count * IB_CQ_SIZE;\r
-\r
-       cl_atomic_inc( &cq_tinfo->qp_count );\r
+       cqe_size = (cq_tinfo->qp_count + 1) * IB_CQ_SIZE;\r
 \r
-       if( cq_tinfo->cqe_size < current_cqe_size )\r
+       if( cq_tinfo->cqe_size < cqe_size )\r
        {\r
-               ib_api_status_t status;\r
-               status = ib_modify_cq( cq_tinfo->cq, &current_cqe_size );\r
-               if( status )\r
-               {\r
-                       /* \r
-                        * TODO: This could mean we are out of cqe and need to have\r
-                        * more than one cq per HCA in the future.\r
-                        */\r
-                       cl_atomic_dec( &cq_tinfo->qp_count );\r
-                       IBSP_ERROR_EXIT(\r
-                               ("ib_modify_cq() failed. (%d)\n", status) );\r
-                       return (NULL);\r
-               }\r
-               else\r
+               status = ib_modify_cq( cq_tinfo->cq, &cqe_size );\r
+               switch( status )\r
                {\r
-                       cq_tinfo->cqe_size = current_cqe_size;\r
+               case IB_INVALID_CQ_SIZE:\r
+                       cq_tinfo = ib_alloc_cq_tinfo( hca );\r
+                       if( !cq_tinfo )\r
+                               break;\r
+\r
+                       cq_tinfo->qp_count++;\r
+                       break;\r
+\r
+               case IB_SUCCESS:\r
+                       cq_tinfo->cqe_size = cqe_size;\r
+\r
+                       cq_tinfo->qp_count++;\r
+\r
                        fzprint(("%s():%d:0x%x:0x%x: New cq size=%d.\n",\r
                                         __FUNCTION__,\r
                                         __LINE__, GetCurrentProcessId(),\r
                                         GetCurrentThreadId(), cq_tinfo->cqe_size));\r
+                       break;\r
 \r
+               default:\r
+                       IBSP_ERROR_EXIT(\r
+                               ("ib_modify_cq() returned %s\n", ib_get_err_str(status)) );\r
+                       cq_tinfo = NULL;\r
                }\r
        }\r
+       else\r
+       {\r
+               cq_tinfo->qp_count++;\r
+       }\r
 \r
+       cl_spinlock_release( &hca->cq_lock );\r
        IBSP_EXIT( IBSP_DBG_HW );\r
-\r
        return (cq_tinfo);\r
 }\r
 \r
@@ -742,9 +738,14 @@ ib_release_cq_tinfo(
 {\r
        IBSP_ENTER( IBSP_DBG_HW );\r
 \r
-       cl_atomic_dec( &cq_tinfo->qp_count );\r
+       CL_ASSERT( cq_tinfo );\r
+       CL_ASSERT( cq_tinfo->hca );\r
 \r
-       /* TODO: downsize the cq  */\r
+       cl_spinlock_acquire( &cq_tinfo->hca->cq_lock );\r
+       /* If this CQ now has fewer QPs than the primary, make it the primary. */\r
+       if( --cq_tinfo->qp_count < cq_tinfo->hca->cq_tinfo->qp_count )\r
+               cq_tinfo->hca->cq_tinfo = cq_tinfo;\r
+       cl_spinlock_release( &cq_tinfo->hca->cq_lock );\r
 \r
        IBSP_EXIT( IBSP_DBG_HW );\r
 }\r
@@ -769,12 +770,6 @@ ib_release(void)
                {\r
                        struct ibsp_hca *hca = PARENT_STRUCT(item, struct ibsp_hca, item);\r
 \r
-                       if( hca->cq_tinfo )\r
-                       {\r
-                               CL_ASSERT( hca->cq_tinfo->qp_count == 0 );\r
-                               ib_destroy_cq_tinfo( hca->cq_tinfo );\r
-                       }\r
-\r
                        pnp_ca_remove( hca );\r
                }\r
 \r
@@ -788,7 +783,8 @@ ib_release(void)
                                 status));\r
                if( status != IB_SUCCESS )\r
                {\r
-                       IBSP_ERROR( ("ib_close_al failed (%d)\n", status) );\r
+                       IBSP_ERROR(\r
+                               ("ib_close_al returned %s\n", ib_get_err_str( status )) );\r
                }\r
                else\r
                {\r
@@ -907,11 +903,8 @@ int
 ib_create_socket(\r
        IN      OUT                     struct ibsp_socket_info         *socket_info)\r
 {\r
-       struct cq_thread_info   *cq_tinfo;\r
        ib_qp_create_t                  qp_create;\r
        ib_api_status_t                 status;\r
-       int                                             ret;\r
-       struct ibsp_hca                 *hca;\r
        ib_qp_attr_t                    qp_attr;\r
 \r
        IBSP_ENTER( IBSP_DBG_EP );\r
@@ -920,18 +913,15 @@ ib_create_socket(
        CL_ASSERT( socket_info->port != NULL );\r
        CL_ASSERT( socket_info->qp == NULL );\r
 \r
-       hca = socket_info->port->hca;\r
-       socket_info->hca_pd = hca->pd;\r
+       socket_info->hca_pd = socket_info->port->hca->pd;\r
 \r
        /* Get the completion queue and thread info for this socket */\r
-       cq_tinfo = ib_acquire_cq_tinfo( hca );\r
-       if( cq_tinfo == NULL )\r
+       socket_info->cq_tinfo = ib_acquire_cq_tinfo( socket_info->port->hca );\r
+       if( !socket_info->cq_tinfo )\r
        {\r
-               IBSP_ERROR( ("ib_acquire_cq_tinfo failed\n") );\r
-               ret = WSAEPROVIDERFAILEDINIT;\r
-               goto done;\r
+               IBSP_ERROR_EXIT( ("ib_acquire_cq_tinfo failed\n") );\r
+               return WSAENOBUFS;\r
        }\r
-       socket_info->cq_tinfo = cq_tinfo;\r
 \r
        /* Queue pair */\r
        qp_create.qp_type = IB_QPT_RELIABLE_CONN;\r
@@ -939,8 +929,8 @@ ib_create_socket(
        qp_create.rq_depth = QP_ATTRIB_RQ_DEPTH;\r
        qp_create.sq_sge = QP_ATTRIB_SQ_SGE;\r
        qp_create.rq_sge = 1;\r
-       qp_create.h_rq_cq = cq_tinfo->cq;\r
-       qp_create.h_sq_cq = cq_tinfo->cq;\r
+       qp_create.h_rq_cq = socket_info->cq_tinfo->cq;\r
+       qp_create.h_sq_cq = socket_info->cq_tinfo->cq;\r
        qp_create.sq_signaled = TRUE;\r
 \r
        status = ib_create_qp( socket_info->hca_pd, &qp_create, socket_info,    /* context */\r
@@ -948,9 +938,9 @@ ib_create_socket(
                &socket_info->qp );\r
        if( status )\r
        {\r
-               IBSP_ERROR( ("ib_create_qp failed (%d)\n", status));\r
-               ret = WSAEPROVIDERFAILEDINIT;\r
-               goto done;\r
+               IBSP_ERROR_EXIT(\r
+                       ("ib_create_qp returned %s\n", ib_get_err_str( status )) );\r
+               return WSAENOBUFS;\r
        }\r
 \r
        status = ib_query_qp( socket_info->qp, &qp_attr );\r
@@ -960,24 +950,14 @@ ib_create_socket(
        }\r
        else\r
        {\r
-               IBSP_ERROR(\r
-                       ("ib_query_qp returned %s\n", ib_get_err_str( status )) );\r
+               IBSP_ERROR( ("ib_query_qp returned %s\n", ib_get_err_str( status )) );\r
                socket_info->max_inline = 0;\r
        }\r
 \r
        STAT_INC( qp_num );\r
 \r
-       ret = 0;\r
-\r
-  done:\r
-       if( ret )\r
-       {\r
-               ib_destroy_socket( socket_info );\r
-       }\r
-\r
        IBSP_EXIT( IBSP_DBG_EP );\r
-\r
-       return ret;\r
+       return 0;\r
 }\r
 \r
 \r

diff --git a/trunk/ulp/wsd/user/ibsp_pnp.c b/trunk/ulp/wsd/user/ibsp_pnp.c
index 77f4074bb3e4e29eb32153e42de25b99b1455325..a968856382798f9513c5df8fd114603c73143123 100644 (file)
--- a/trunk/ulp/wsd/user/ibsp_pnp.c
+++ b/trunk/ulp/wsd/user/ibsp_pnp.c
@@ -91,6 +91,7 @@ pnp_ca_add(
        cl_spinlock_init( &hca->port_lock );\r
        cl_qlist_init( &hca->rdma_mem_list.list );\r
        cl_spinlock_init( &hca->rdma_mem_list.mutex );\r
+       cl_spinlock_init( &hca->cq_lock );\r
 \r
        /* HCA handle */\r
        IBSP_TRACE( IBSP_DBG_HW,\r
@@ -143,8 +144,9 @@ void
 pnp_ca_remove(\r
                                        struct ibsp_hca                         *hca )\r
 {\r
-       ib_api_status_t status;\r
-       cl_list_item_t  *p_item;\r
+       ib_api_status_t                 status;\r
+       cl_list_item_t                  *p_item;\r
+       struct cq_thread_info   *p_cq_tinfo;\r
 \r
        IBSP_ENTER( IBSP_DBG_HW );\r
 \r
@@ -161,6 +163,26 @@ pnp_ca_remove(
        }\r
        cl_spinlock_release( &hca->port_lock );\r
 \r
+       cl_spinlock_acquire( &hca->cq_lock );\r
+       while( hca->cq_tinfo )\r
+       {\r
+               p_cq_tinfo = hca->cq_tinfo;\r
+\r
+               hca->cq_tinfo = PARENT_STRUCT(\r
+                       cl_qlist_next( &hca->cq_tinfo->list_item ),\r
+               struct cq_thread_info, list_item );\r
+\r
+               __cl_primitive_remove( &p_cq_tinfo->list_item );\r
+\r
+               if( hca->cq_tinfo == p_cq_tinfo )\r
+                       break;\r
+\r
+               cl_spinlock_release( &hca->cq_lock );\r
+               ib_destroy_cq_tinfo( hca->cq_tinfo );\r
+               cl_spinlock_acquire( &hca->cq_lock );\r
+       }\r
+       cl_spinlock_release( &hca->cq_lock );\r
+\r
        if( hca->pd )\r
        {\r
                ib_deregister_all_mr( &hca->rdma_mem_list );\r
@@ -195,6 +217,8 @@ pnp_ca_remove(
        cl_spinlock_destroy( &hca->port_lock );\r
        cl_spinlock_destroy( &hca->rdma_mem_list.mutex );\r
 \r
+       cl_spinlock_destroy( &hca->cq_lock );\r
+\r
        HeapFree( g_ibsp.heap, 0, hca );\r
 \r
        IBSP_EXIT( IBSP_DBG_HW );\r

diff --git a/trunk/ulp/wsd/user/ibspdefines.h b/trunk/ulp/wsd/user/ibspdefines.h
index eea39708aafc2de215f55055c12f8c04b0e43485..859b625793eee8874901afd99638b79b11742146 100644 (file)
--- a/trunk/ulp/wsd/user/ibspdefines.h
+++ b/trunk/ulp/wsd/user/ibspdefines.h
@@ -72,7 +72,7 @@ C_ASSERT( QP_ATTRIB_SQ_DEPTH <= 256 );
 C_ASSERT( QP_ATTRIB_RQ_DEPTH <= 256 );\r
 \r
 /* Number of entries in a CQ */\r
-#define IB_CQ_SIZE (QP_ATTRIB_SQ_DEPTH + QP_ATTRIB_RQ_DEPTH + 1)\r
+#define IB_CQ_SIZE (QP_ATTRIB_SQ_DEPTH + QP_ATTRIB_RQ_DEPTH)\r
 \r
 /* CM timeouts */\r
 #define CM_MIN_LOCAL_TIMEOUT   (18)\r

diff --git a/trunk/ulp/wsd/user/ibspproto.h b/trunk/ulp/wsd/user/ibspproto.h
index 1fc338e5da4d287913582370881704ae2ded5df2..fa0a102932055626f09d23b68b388ebbbd41f130 100644 (file)
--- a/trunk/ulp/wsd/user/ibspproto.h
+++ b/trunk/ulp/wsd/user/ibspproto.h
@@ -116,6 +116,13 @@ ib_release( void );
 extern int\r
 ibsp_initialize( void );\r
 \r
+void\r
+ib_release_cq_tinfo(\r
+                                       struct cq_thread_info           *cq_tinfo );\r
+void\r
+ib_destroy_cq_tinfo(\r
+                                       struct cq_thread_info           *cq_tinfo );\r
+\r
 int\r
 ib_create_socket(\r
        IN      OUT                     struct ibsp_socket_info         *socket_info );\r

diff --git a/trunk/ulp/wsd/user/ibspstruct.h b/trunk/ulp/wsd/user/ibspstruct.h
index 4588174a8237881d03b4421576cd27e91a2d99ad..59316c44c6903e037df4f49b5a2d3dfca9fb6939 100644 (file)
--- a/trunk/ulp/wsd/user/ibspstruct.h
+++ b/trunk/ulp/wsd/user/ibspstruct.h
@@ -183,15 +183,11 @@ struct rdma_memory_desc
 \r
 struct cq_thread_info\r
 {\r
-       /* For future growth if the hca needs a list of cqs */\r
-       cl_list_item_t item;\r
+       cl_list_item_t          list_item;\r
 \r
        cl_waitobj_handle_t cq_waitobj;\r
        ib_cq_handle_t cq;\r
 \r
-       cl_spinlock_t wr_mutex;\r
-       cl_qlist_t done_wr_list;\r
-\r
        /* Number of qp's using this cq */\r
        atomic32_t qp_count;\r
 \r
@@ -391,7 +387,11 @@ struct ibsp_hca
        cl_spinlock_t   port_lock;\r
        cl_qlist_t              port_list;\r
 \r
-       /* TODO: Make this a dynamic list if we need more than one cq_tinfo per HCA */\r
+       /*\r
+        * The CQ list is a circular list without an end.  The pointer here\r
+        * points to the entry that should be used for the next allocation.\r
+        */\r
+       cl_spinlock_t   cq_lock;\r
        struct cq_thread_info *cq_tinfo;\r
 };\r
 \r