]> git.openfabrics.org - ~shefty/rdma-win.git/commitdiff
[IBAL] 1) fixed a bug in RTU handling, which caused immediate unbinding QP from CEP...
authorleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sun, 25 Nov 2007 15:30:39 +0000 (15:30 +0000)
committerleonidk <leonidk@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sun, 25 Nov 2007 15:30:39 +0000 (15:30 +0000)
2) fixed a bug with not-releasing of a parameter  structure, allocated for REQ handling, in several paths of error flow;
3) fixed incorrect ioctl return code;
4) reworked usage of reference counting to make the code more readable;
5) added AL_RESERVED_CID. It is set to enforce CEP destroy before destroying the bound QP;
6) fixed some CL_ASSERTs to adopt setting a new AL_RESERVED_CID value upon QP destroy;
7) added more debug prints;

git-svn-id: svn://openib.tc.cornell.edu/gen1@908 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

branches/Ndi/core/al/al_cm_cep.h
branches/Ndi/core/al/al_cm_qp.c
branches/Ndi/core/al/al_dev.h
branches/Ndi/core/al/al_qp.c
branches/Ndi/core/al/kernel/al_cm_cep.c
branches/Ndi/core/al/kernel/al_ndi_cm.c
branches/Ndi/core/al/kernel/al_ndi_cm.h
branches/Ndi/core/al/kernel/al_proxy_cep.c
branches/Ndi/core/al/kernel/al_proxy_ndi.c

index 76f6c0f8d03e83536b539f908f79aee0a8f1d6d7..9d067852c1d6f70574688f2e405be35fb0081233 100644 (file)
@@ -55,6 +55,7 @@
 \r
 \r
 #define AL_INVALID_CID         0xFFFFFFFF\r
+#define AL_RESERVED_CID                0\r
 \r
 \r
 typedef void\r
index 187a81da63a21cb12bc9d07fde4fb65187377eae..468216a77eff362313e0b71535ac6a499fe694d5 100644 (file)
@@ -528,7 +528,7 @@ __proc_rej(
                __cep_timewait_qp( p_cm->h_qp );\r
 \r
                cid = cl_atomic_xchg( &((al_conn_qp_t*)p_cm->h_qp)->cid, AL_INVALID_CID );\r
-               CL_ASSERT( cid == p_cm->cid || cid == AL_INVALID_CID );\r
+               CL_ASSERT( cid == p_cm->cid || cid == AL_INVALID_CID || cid == AL_RESERVED_CID );\r
                if( cid == AL_INVALID_CID ||\r
                        al_destroy_cep( p_cm->h_al, cid, deref_al_obj ) != IB_SUCCESS )\r
                {\r
@@ -788,12 +788,14 @@ __proc_conn(
        {\r
        case CM_REQ_ATTR_ID:\r
                CL_ASSERT( ((al_conn_qp_t*)p_cm->h_qp)->cid == (int32_t)p_cm->cid ||\r
+                       ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_RESERVED_CID ||\r
                        ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_INVALID_CID );\r
                __proc_peer_req( p_cm, (mad_cm_req_t*)p_mad );\r
                break;\r
 \r
        case CM_MRA_ATTR_ID:\r
                CL_ASSERT( ((al_conn_qp_t*)p_cm->h_qp)->cid == (int32_t)p_cm->cid ||\r
+                       ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_RESERVED_CID ||\r
                        ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_INVALID_CID );\r
                __proc_mra( p_cm, (mad_cm_mra_t*)p_mad );\r
                break;\r
@@ -804,12 +806,14 @@ __proc_conn(
 \r
        case CM_REP_ATTR_ID:\r
                CL_ASSERT( ((al_conn_qp_t*)p_cm->h_qp)->cid == (int32_t)p_cm->cid ||\r
+                       ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_RESERVED_CID ||\r
                        ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_INVALID_CID );\r
                __proc_rep( p_cm, (mad_cm_rep_t*)p_mad );\r
                break;\r
 \r
        case CM_RTU_ATTR_ID:\r
                CL_ASSERT( ((al_conn_qp_t*)p_cm->h_qp)->cid == (int32_t)p_cm->cid ||\r
+                       ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_RESERVED_CID ||\r
                        ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_INVALID_CID );\r
                __proc_rtu( p_cm, (mad_cm_rtu_t*)p_mad );\r
                break;\r
@@ -822,12 +826,14 @@ __proc_conn(
 \r
        case CM_DREP_ATTR_ID:\r
                CL_ASSERT( ((al_conn_qp_t*)p_cm->h_qp)->cid == (int32_t)p_cm->cid ||\r
+                       ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_RESERVED_CID ||\r
                        ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_INVALID_CID );\r
                __proc_drep( p_cm, (mad_cm_drep_t*)p_mad );\r
                break;\r
 \r
        case CM_LAP_ATTR_ID:\r
                CL_ASSERT( ((al_conn_qp_t*)p_cm->h_qp)->cid == (int32_t)p_cm->cid ||\r
+                       ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_RESERVED_CID ||\r
                        ((al_conn_qp_t*)p_cm->h_qp)->cid == AL_INVALID_CID );\r
                __proc_lap( p_cm, (mad_cm_lap_t*)p_mad );\r
                break;\r
index 5ada0c7c5107e342a8a62ef40be4cceffb64e14e..cca9874bbefe72c978b578d487e8e84083b83752 100644 (file)
@@ -54,7 +54,7 @@
 #define AL_DEVICE_NAME L"\\Device\\ibal"\r
 #define        ALDEV_KEY               (0x3B)  /* Matches FILE_DEVICE_INFINIBAND from wdm.h */\r
 \r
-#define AL_IOCTL_VERSION                       (3)\r
+#define AL_IOCTL_VERSION                       (4)\r
 \r
 \r
 #ifdef CL_KERNEL\r
index eecb686a81fc7e1e8a8ea8926147910f0c700c1f..1edf3600de3a461f5a31de5b6c77edc7701edb96 100644 (file)
@@ -1118,7 +1118,7 @@ destroying_qp(
        case IB_QPT_RELIABLE_CONN:\r
        case IB_QPT_UNRELIABLE_CONN:\r
                cid = cl_atomic_xchg(\r
-                       &((al_conn_qp_t*)h_qp)->cid, AL_INVALID_CID );\r
+                       &((al_conn_qp_t*)h_qp)->cid, AL_RESERVED_CID );\r
                if( cid != AL_INVALID_CID )\r
                {\r
                        ref_al_obj( &h_qp->obj );\r
index 9cc886c4bbd160488cc438320922c8c0809180a6..56556d56d552cdad272d5fee4a3d66bd6ad1e7d5 100644 (file)
@@ -3403,9 +3403,9 @@ __cep_set_pdata(
        cl_memclr( p_cep->pdata, sizeof(p_cep->pdata) );\r
        p_cep->psize = min( psize, sizeof(p_cep->pdata) );\r
        memcpy( p_cep->pdata, pdata, p_cep->psize );\r
-       AL_PRINT(TRACE_LEVEL_ERROR ,AL_DBG_ERROR ,\r
-               ("__cep_set_pdata: set %d of pdata for cid %d, h_al %p \n", \r
-               p_cep->psize, cid, h_al ));\r
+       AL_PRINT(TRACE_LEVEL_INFORMATION ,AL_DBG_CM ,\r
+               ("__cep_set_pdata: set %d of pdata for cid %d, h_al %p, context %p \n", \r
+               p_cep->psize, cid, h_al, p_cep->context ));\r
 \r
        AL_EXIT( AL_DBG_CM );\r
        return IB_SUCCESS;\r
@@ -4112,6 +4112,10 @@ al_create_cep(
 \r
        *p_cid = p_cep->cid;\r
 \r
+       AL_PRINT(TRACE_LEVEL_INFORMATION ,AL_DBG_CM ,\r
+               ("Created CEP with cid %d, h_al %p, context %p \n", \r
+               p_cep->cid, h_al, p_cep->context ));\r
+\r
        AL_EXIT( AL_DBG_CM );\r
        return IB_SUCCESS;\r
 }\r
@@ -4163,6 +4167,10 @@ al_destroy_cep(
        if( !ref_cnt && pfn_destroy_cb )\r
                pfn_destroy_cb( context );\r
 \r
+       AL_PRINT(TRACE_LEVEL_INFORMATION ,AL_DBG_CM ,\r
+               ("Destroeyd CEP with cid %d, h_al %p, context %p \n", \r
+               cid, h_al, context ));\r
+\r
        AL_EXIT( AL_DBG_CM );\r
        return IB_SUCCESS;\r
 }\r
@@ -6423,7 +6431,8 @@ al_cep_get_pdata(
        }\r
 \r
        AL_PRINT(TRACE_LEVEL_INFORMATION ,AL_DBG_CM ,\r
-               ("al_cep_get_pdata: get %d of pdata from CEP with cid %d \n", p_cep->psize, cid ));\r
+               ("al_cep_get_pdata: get %d of pdata from CEP with cid %d, h_al %p, context %p \n", \r
+               p_cep->psize, cid, h_al, p_cep->context ));\r
 \r
        KeReleaseInStackQueuedSpinLock( &hdl );\r
        AL_EXIT( AL_DBG_CM );\r
index 85c3892beeb267a30ccbe5f6d6881ad73c4b6503..6fcafab91f92ce79203d9c5a6e831e7f340cdc36 100644 (file)
@@ -96,14 +96,13 @@ __ndi_complete_irp(
        )
 {
        AL_ENTER( AL_DBG_NDI );
-       if (Irp)
-       {
-               cl_ioctl_complete( Irp, code, 0 );
-               deref_al_obj( &h_qp->obj );
-               h_qp->p_irp_que->h_ioctl = NULL;        /* mark IRP as cancelled */
-       }
-       else
-               AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("Irp is NULL - not completed !\n") );
+
+       CL_ASSERT( Irp );
+
+       cl_ioctl_complete( Irp, code, 0 );
+       deref_al_obj( &h_qp->obj );                     /* release IRP life reference */
+       h_qp->p_irp_que->h_ioctl = NULL;        /* mark IRP as cancelled */
+
        AL_EXIT( AL_DBG_NDI );
 }
 
@@ -119,8 +118,8 @@ __ndi_complete_irp_ex(
        AL_ENTER( AL_DBG_NDI );
        h_qp->p_irp_que->state = new_state;     
        Irp = IoCsqRemoveNextIrp( &h_qp->p_irp_que->csq, NULL );
-       __ndi_complete_irp( h_qp, Irp, code );
-       deref_al_obj( &h_qp->obj );     /* release the extra reference */
+       if ( Irp )
+               __ndi_complete_irp( h_qp, Irp, code );
        AL_EXIT( AL_DBG_NDI );
 }
 
@@ -402,10 +401,11 @@ static inline void __ndi_flush_que(
        while( Irp = IoCsqRemoveNextIrp( &p_ndi_csq->csq, NULL ) )
        {
                AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI, 
-                       ("h_qp %I64x, ref_cnt %d\n", (uint64_t)p_ndi_csq->h_qp, 
+                       ("h_qp %#I64x, uhdl %#I64x, ref_cnt %d\n", 
+                       (uint64_t)p_ndi_csq->h_qp, p_ndi_csq->h_qp->obj.hdl, 
                        p_ndi_csq->h_qp->obj.ref_cnt ) );
                cl_ioctl_complete( Irp, completion_code, 0 );
-               deref_al_obj( &p_ndi_csq->h_qp->obj );
+               deref_al_obj( &p_ndi_csq->h_qp->obj );          /* release IRP life reference */
        }
 }
 #pragma warning(default:4706)
@@ -418,7 +418,8 @@ ndi_qp_flush_ques(
        AL_ENTER( AL_DBG_NDI );
        __ndi_flush_que( h_qp->p_irp_que, STATUS_CANCELLED );
        AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI, 
-               ("h_qp %I64x, ref_cnt %d\n", (uint64_t)h_qp, h_qp->obj.ref_cnt ) );
+               ("h_qp %#I64x, uhdl %#I64x, ref_cnt %d\n", 
+               (uint64_t)h_qp, h_qp->obj.hdl, h_qp->obj.ref_cnt ) );
        AL_EXIT( AL_DBG_NDI );
 }
 
@@ -433,6 +434,12 @@ ndi_qp_init(
 
        AL_ENTER( AL_DBG_NDI );
 
+       if ( h_qp->type == IB_QPT_UNRELIABLE_CONN )
+       {
+               status = STATUS_NOT_SUPPORTED;
+               goto exit;
+       }
+       
        h_qp->p_irp_que = (ndi_qp_csq_t*)cl_zalloc(sizeof(ndi_qp_csq_t));
        if (!h_qp->p_irp_que)
        {
@@ -453,6 +460,10 @@ ndi_qp_init(
        h_qp->p_irp_que->state = NDI_CM_IDLE;
        status = STATUS_SUCCESS;
 
+AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+       ("Creating h_qp %#I64x, uhdl %#I64x \n", 
+       (uint64_t)h_qp, h_qp->obj.hdl ) );
+
 exit:
        AL_EXIT( AL_DBG_NDI );
        return status;
@@ -467,8 +478,8 @@ ndi_qp_destroy(
        if (h_qp->type == IB_QPT_RELIABLE_CONN && h_qp->p_irp_que)
        {
                AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
-                       ("Destroying h_qp %I64x. h_ioctl %p\n", 
-                       (uint64_t)h_qp, h_qp->p_irp_que->h_ioctl ) );
+                       ("Destroying h_qp %#I64x, uhdl %#I64x, h_ioctl %p, cid %d\n", 
+                       (uint64_t)h_qp, h_qp->obj.hdl, h_qp->p_irp_que->h_ioctl, ((al_conn_qp_t*)h_qp)->cid ) );
 
                /* cancel pending IRPS for NDI type CQ */
                ndi_qp_flush_ques( h_qp );
@@ -505,7 +516,6 @@ __ndi_req_cm_wi(
        IN                              DEVICE_OBJECT*                          p_dev_obj,
        IN                              void*                                           context )
 {
-       net32_t cid;
        NTSTATUS status;
        ib_qp_handle_t  h_qp = (ib_qp_handle_t)context;
        UNUSED_PARAM(p_dev_obj);
@@ -516,19 +526,10 @@ __ndi_req_cm_wi(
 
        __cep_timewait_qp( h_qp );
        
-       cid = cl_atomic_xchg(
-               &((al_conn_qp_t*)h_qp)->cid, AL_INVALID_CID );
-       ref_al_obj( &h_qp->obj );
-       if( cid == AL_INVALID_CID || al_destroy_cep(
-               h_qp->obj.h_al, cid, deref_al_obj ) != IB_SUCCESS )
-       {
-               deref_al_obj( &h_qp->obj );
-       }
-
        h_qp->p_irp_que->state = NDI_CM_IDLE;   
        status = (h_qp->p_irp_que->state == NDI_CM_CONNECTED) ? STATUS_CONNECTION_DISCONNECTED : STATUS_CONNECTION_REFUSED;
        __ndi_complete_irp( h_qp, h_qp->p_irp_que->h_ioctl, status );
-       deref_al_obj( &h_qp->obj );     /* release the extra reference */
+       deref_al_obj( &h_qp->obj );     /* release IRP SA reference */
 
        AL_EXIT( AL_DBG_NDI );
 }
@@ -539,39 +540,40 @@ __ndi_req_cm_wi(
  */
        
        
-void
+static void
 __ndi_proc_rej(
        IN              const   ib_cm_handle_t* const           p_cm,
        IN              const   mad_cm_rej_t* const             p_rej )
 {
        net32_t cid;
        NTSTATUS status;
-       ndi_qp_csq_t *p_ndi_csq = p_cm->h_qp->p_irp_que;
+       ib_qp_handle_t  h_qp = p_cm->h_qp;
+       ndi_qp_csq_t *p_ndi_csq = h_qp->p_irp_que;
 
        AL_ENTER( AL_DBG_NDI );
 
-       AL_PRINT(TRACE_LEVEL_INFORMATION, AL_DBG_NDI
-               ("p_rej %p, h_qp %p connect reject, reason=%d\n", 
-               p_rej, p_cm->h_qp, cl_ntoh16(p_rej->reason) ) );
+       AL_PRINT(TRACE_LEVEL_ERROR, AL_DBG_ERROR
+               ("p_rej %p, h_qp %#I64x, uhdl %#I64x, connect reject, reason=%hd\n", 
+               p_rej, (uint64_t)h_qp, h_qp->obj.hdl, cl_ntoh16(p_rej->reason) ) );
 
-       ref_al_obj( &p_cm->h_qp->obj );
-       cid = cl_atomic_xchg( &((al_conn_qp_t*)p_cm->h_qp)->cid, AL_INVALID_CID );
-       CL_ASSERT( cid == p_cm->cid || cid == AL_INVALID_CID );
+       ref_al_obj( &h_qp->obj );
+       cid = cl_atomic_xchg( &((al_conn_qp_t*)h_qp)->cid, AL_INVALID_CID );
+       CL_ASSERT( cid == p_cm->cid || cid == AL_RESERVED_CID || cid == AL_INVALID_CID );
        if( cid == AL_INVALID_CID ||
                al_destroy_cep( p_cm->h_al, cid, deref_al_obj ) != IB_SUCCESS )
        {
-               deref_al_obj( &p_cm->h_qp->obj );
+               deref_al_obj( &h_qp->obj );
        }
 
        switch (p_ndi_csq->state)
        {
        case NDI_CM_CONNECTING_REQ_SENT:
                al_cep_set_pdata( p_cm->h_al, p_cm->cid, IB_REJ_PDATA_SIZE, (uint8_t*)p_rej->pdata );
-               AL_PRINT(TRACE_LEVEL_ERROR ,AL_DBG_ERROR ,
+               AL_PRINT(TRACE_LEVEL_INFORMATION, AL_DBG_NDI ,
                        ("set %d of REQ pdata to CEP with cid %d, h_al %p\n", 
                        IB_REJ_PDATA_SIZE, p_cm->cid, p_cm->h_al ));
                status = (p_rej->reason == IB_REJ_TIMEOUT) ? STATUS_TIMEOUT : STATUS_CONNECTION_REFUSED;
-               __ndi_complete_irp_ex( p_cm->h_qp, status, NDI_CM_CONNECTING_REJ_RCVD );
+               __ndi_complete_irp_ex( h_qp, status, NDI_CM_CONNECTING_REJ_RCVD );
                break;
 
        case NDI_CM_CONNECTED:
@@ -580,36 +582,41 @@ __ndi_proc_rej(
                   We are treating this case it like a DREQ */
                {
                        IO_STACK_LOCATION *p_io_stack;
-                       PIO_WORKITEM p_workitem;
                        cl_ioctl_handle_t h_ioctl;
 
-                       h_ioctl = IoCsqRemoveNextIrp( &p_cm->h_qp->p_irp_que->csq, NULL );
+                       h_ioctl = IoCsqRemoveNextIrp( &h_qp->p_irp_que->csq, NULL );
                        if (!h_ioctl)
                        { /* IRP has been cancelled */
-                               //TODO: don't know where to take DeviceObject */
-                               AL_PRINT(TRACE_LEVEL_INFORMATION, AL_DBG_NDI, 
-                                       ("Not handled REJ case: ndi_state %d\n", 
-                                       p_ndi_csq->state ) );
+                               // TODO: no QP flash
+                               AL_PRINT(TRACE_LEVEL_INFORMATION, AL_DBG_NDI ,
+                                       ( "IRP cancelled: Can't flush the QP %p, ndi_state %d\n",
+                                       h_qp, p_ndi_csq->state ) );
+                               h_qp->p_irp_que->state = NDI_CM_IDLE;   
 //                             ASSERT(FALSE);
-                               break;
-                       }
-                       p_io_stack = IoGetCurrentIrpStackLocation( h_ioctl );
-                       p_workitem = IoAllocateWorkItem( p_io_stack->DeviceObject );
-                       if ( p_workitem )
-                       { /* asyncronous performing */
-                               p_cm->h_qp->p_irp_que->p_workitem = p_workitem;
-                               IoQueueWorkItem( p_workitem, __ndi_req_cm_wi, DelayedWorkQueue, p_cm->h_qp );
-                               break;
                        }
                        else
-                       { /* syncronous performing */
-                               //TODO: what can we do here ?
-                               AL_PRINT(TRACE_LEVEL_INFORMATION, AL_DBG_NDI, 
-                                       ("Not handled REJ case: ndi_state %d\n", 
-                                       p_ndi_csq->state ) );
-//                             ASSERT(FALSE);
-                               break;
+                       {
+                               p_io_stack = IoGetCurrentIrpStackLocation( h_ioctl );
+                               h_qp->p_irp_que->p_workitem = IoAllocateWorkItem( p_io_stack->DeviceObject );
+                               if ( h_qp->p_irp_que->p_workitem )
+                               { /* asyncronous performing */
+                                       IoQueueWorkItem( h_qp->p_irp_que->p_workitem,
+                                               __ndi_req_cm_wi, DelayedWorkQueue, h_qp );
+                               }
+                               else
+                               { /* syncronous performing */
+                                       // TODO: no QP flash
+                                       AL_PRINT(TRACE_LEVEL_ERROR ,AL_DBG_ERROR ,
+                                               ( "Can't flush the QP %p, ndi_state %d\n",
+                                               h_qp, p_ndi_csq->state ) );
+                                       h_qp->p_irp_que->state = NDI_CM_IDLE;   
+                                       status = (h_qp->p_irp_que->state == NDI_CM_CONNECTED) ? STATUS_CONNECTION_DISCONNECTED : STATUS_CONNECTION_REFUSED;
+                                       __ndi_complete_irp( h_qp, h_ioctl, status );
+                                       deref_al_obj( &h_qp->obj );     /* release IRP SA reference */
+                                       h_qp->p_irp_que->h_ioctl = NULL;
+                               }
                        }
+                       break;
                }
                
        default:        
@@ -633,7 +640,7 @@ __ndi_proc_rep(
 
        AL_ENTER( AL_DBG_NDI );
 
-       AL_PRINT(TRACE_LEVEL_INFORMATION ,AL_DBG_NDI ,("h_qp = 0x%p\n", p_cm->h_qp));
+       AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI ,("h_qp = 0x%p\n", p_cm->h_qp));
 
        if ( p_ndi_csq->state  != NDI_CM_CONNECTING_REQ_SENT) 
        {
@@ -751,68 +758,65 @@ __ndi_cm_handler(
                ib_mad_t*p_mad = ib_get_mad_buf( p_mad_el );
                ib_qp_handle_t h_qp = (ib_qp_handle_t)context;
 
-               h_cm.h_al = h_al;
-               h_cm.h_qp = h_qp;
-               h_cm.cid = cid;
-
                if( p_mad_el->status != IB_SUCCESS )
                {
                        switch( p_mad->attr_id )
                        {
                        case CM_REQ_ATTR_ID:
+                               AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+                                       ("REQ timeouted for CEP with cid %d, h_al %p, context %p, new_cid %d .\n", 
+                                       cid, h_al, h_qp, new_cid ) );
                                __ndi_complete_irp_ex( h_qp, STATUS_TIMEOUT, NDI_CM_IDLE );
                                break;
 
                        case CM_REP_ATTR_ID:
+                               AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+                                       ("REP timeouted for CEP with cid %d, h_al %p, context %p, new_cid %d .\n", 
+                                       cid, h_al, h_qp, new_cid ) );
                                __ndi_complete_irp_ex( h_qp, STATUS_CONNECTION_ABORTED, NDI_CM_IDLE );
                                break;
 
                        case CM_DREQ_ATTR_ID:
-                               __ndi_complete_irp_ex( h_qp, STATUS_SUCCESS, NDI_CM_IDLE );
-                               break;
-
                        default:
                                AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
-                                       ("Invalid CM send MAD attribute ID %d.\n", p_mad->attr_id) );
+                                       ("Unhandled MAD attr ID %d for CEP with cid %d, h_al %p, context %p, new_cid %d .\n", 
+                                       p_mad->attr_id, cid, h_al, h_qp, new_cid ) );
                                break;
                        }
                }
                else
                {
+                       h_cm.h_al = h_al;
+                       h_cm.h_qp = h_qp;
+                       h_cm.cid = cid;
+                       
                        switch( p_mad->attr_id )
                        {
                        case CM_REP_ATTR_ID:
-                               // TODO: how to prevent calling this handler with QP destroyed ?
-                               if( ((al_conn_qp_t*)h_qp)->cid == (int32_t)cid || 
-                                       ((al_conn_qp_t*)h_qp)->cid == AL_INVALID_CID )
-                               {
-                                       __ndi_proc_rep( &h_cm, (mad_cm_rep_t*)p_mad );
-                               }
-                               else
-                               {
-                                       AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
-                                               ("REP received when QP destroyed, cid %d:%d .\n", 
-                                               ((al_conn_qp_t*)h_qp)->cid, cid ) );
-                               }
+                               AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+                                       ("REP received for CEP with cid %d, h_al %p, context %p, new_cid %d .\n", 
+                                       cid, h_al, h_qp, new_cid ) );
+                               CL_ASSERT( ((al_conn_qp_t*)h_qp)->cid == (int32_t)cid || 
+                                       ((al_conn_qp_t*)h_qp)->cid == AL_INVALID_CID ||
+                                       ((al_conn_qp_t*)h_qp)->cid == AL_RESERVED_CID );
+                               __ndi_proc_rep( &h_cm, (mad_cm_rep_t*)p_mad );
                                break;
                        
                        case CM_REJ_ATTR_ID:
+                               AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+                                       ("REJ received for CEP with cid %d, h_al %p, context %p, new_cid %d .\n", 
+                                       cid, h_al, h_qp, new_cid ) );
                                __ndi_proc_rej( &h_cm, (mad_cm_rej_t*)p_mad );
                                break;
                
                        case CM_DREQ_ATTR_ID:
-                               // TODO: how to prevent calling this handler with QP destroyed ?
-                               if( ((al_conn_qp_t*)h_qp)->cid == (int32_t)cid || 
-                                       ((al_conn_qp_t*)h_qp)->cid == AL_INVALID_CID )
-                               {
-                                       __ndi_proc_dreq( &h_cm, (mad_cm_dreq_t*)p_mad );
-                               }
-                               else
-                               {
-                                       AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
-                                               ("DREQ received when QP destroyed, cid %d:%d .\n", 
-                                               ((al_conn_qp_t*)h_qp)->cid, cid ) );
-                               }
+                               AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+                                       ("DREQ received for CEP with cid %d, h_al %p, context %p, new_cid %d .\n", 
+                                       cid, h_al, h_qp, new_cid ) );
+                               CL_ASSERT( ((al_conn_qp_t*)h_qp)->cid == (int32_t)cid || 
+                               ((al_conn_qp_t*)h_qp)->cid == AL_INVALID_CID ||
+                               ((al_conn_qp_t*)h_qp)->cid == AL_RESERVED_CID );
+                               __ndi_proc_dreq( &h_cm, (mad_cm_dreq_t*)p_mad );
                                break;
                
                        default:
@@ -890,7 +894,7 @@ __ndi_pr_query_cb(
 
        AL_ENTER( AL_DBG_NDI );
 
-       AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
+       AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
                ("status is %d, count is %d, context %p\n", p_query_rec->status,
                p_query_rec->result_cnt, p_query_rec->query_context) );
 
@@ -918,10 +922,13 @@ __ndi_pr_query_cb(
                        ("al_create_cep returned %s.\n", ib_get_err_str( status )) );
                goto err_irp_complete;
        }
+       AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+               ("Created Active CEP with cid %d, h_al %p, context %p\n",
+               cid, qp_get_al( h_qp ), h_qp ) );
 
        /* See if this QP has already been connected. */
        old_cid = cl_atomic_comp_xchg( &((al_conn_qp_t*)h_qp)->cid, AL_INVALID_CID, cid );
-       if( old_cid != AL_INVALID_CID )
+       if( old_cid != AL_INVALID_CID || h_qp->obj.state == CL_DESTROYING )
                goto err_cep_destroy;
 
        /* Format ib_cm_req_t structure */
@@ -946,6 +953,9 @@ __ndi_pr_query_cb(
        if( status != IB_SUCCESS )
                goto err_irp_remove;
 
+       /* we can release it now. In case of QP destroy __destroying_qp will remove CEP */
+       deref_al_obj( &h_qp->obj );     /* release IRP SA reference */
+
        /* SUCCESS ! */
        goto exit;
 
@@ -957,8 +967,10 @@ err_cep_destroy:
        
 err_irp_complete:
        h_qp->p_irp_que->state = NDI_CM_IDLE;   
-       __ndi_complete_irp( h_qp, h_ioctl, STATUS_HOST_UNREACHABLE );
-       deref_al_obj( &h_qp->obj );     /* release the extra reference */
+       /* the IRP "has" 2 QP references, taken in __ndi_ats_query */
+       if ( h_ioctl )
+               __ndi_complete_irp( h_qp, h_ioctl, STATUS_HOST_UNREACHABLE );
+       deref_al_obj( &h_qp->obj );     /* release IRP SA reference */
 
 exit:  
        cl_free( p_req );
@@ -997,7 +1009,7 @@ __ndi_pr_query(
        user_query.src_gid.unicast.interface_id = p_req->guid;
        user_query.dest_gid = *p_dest_gid;
 
-       AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
+       AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
                ("Query for path from %I64x to %I64x\n",
                p_req->guid, ib_gid_get_guid( p_dest_gid )) );
 
@@ -1010,7 +1022,10 @@ __ndi_pr_query(
        if( status != IB_SUCCESS )
        {
                AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("ib_query failed (%d)\n", status) );
+               cl_free( p_req );
                __ndi_complete_irp_ex( h_qp, CL_ERROR, NDI_CM_IDLE );
+               /* relase additional reference, taken in __ndi_ats_query */
+               deref_al_obj( &h_qp->obj );     /* release IRP SA reference */
        }
 
        AL_EXIT( AL_DBG_NDI );
@@ -1036,8 +1051,9 @@ __ndi_ats_query_cb(
        if( !h_ioctl || p_query_rec->status != IB_SUCCESS || !p_query_rec->result_cnt )
        {
                h_qp->p_irp_que->state = NDI_CM_IDLE; 
-               __ndi_complete_irp( h_qp, h_ioctl, STATUS_HOST_UNREACHABLE );
-               deref_al_obj( &h_qp->obj );     /* release the extra reference */
+               if ( h_ioctl )
+                       __ndi_complete_irp( h_qp, h_ioctl, STATUS_HOST_UNREACHABLE );
+               deref_al_obj( &h_qp->obj );     /* release IRP SA reference */
                cl_free( p_req );
                goto exit;              /* ATS request failed */
        }
@@ -1053,8 +1069,8 @@ exit:
        AL_EXIT( AL_DBG_NDI );
 }
 
-/* Synchronously query the SA for a GUID. Return 0 on success. */
-cl_status_t
+/* Send asynchronous query to SA for a GUID. Return STATUS_PENDING on success. */
+NTSTATUS
 __ndi_ats_query(
        IN              ib_qp_handle_t  const                           h_qp,
        IN              cl_ioctl_handle_t                                       h_ioctl,
@@ -1064,7 +1080,7 @@ __ndi_ats_query(
        ib_service_record_t service_record;
        ib_query_req_t query_req;
        ib_api_status_t status;
-       cl_status_t cl_status = CL_PENDING;
+       NTSTATUS nt_status = STATUS_PENDING;
 
        AL_ENTER( AL_DBG_NDI );
 
@@ -1111,26 +1127,35 @@ __ndi_ats_query(
                service_record.service_data8[15]
                ) );
 
-       h_qp->p_irp_que->state = NDI_CM_CONNECTING_ATS_SENT;
-       /* prevent destroying QP after cancelling of the IRP and before ib_query calback*/
        AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI, 
-               ("h_qp %I64x, ref_cnt %d\n", (uint64_t)h_qp, h_qp->obj.ref_cnt ) );
-       ref_al_obj( &h_qp->obj );               /* take the extra reference */
+               ("h_qp %#I64x, uhdl %#I64x, ref_cnt %d\n", 
+               (uint64_t)h_qp, h_qp->obj.hdl, h_qp->obj.ref_cnt ) );
+
+       // preserve QP for the life of this IOCTL
+       ref_al_obj( &h_qp->obj );               /* take IRP life reference */
 
        /* insert IRP in the queue */
        IoCsqInsertIrp( &h_qp->p_irp_que->csq, h_ioctl, NULL );
 
+       /* prevent destroying QP after cancelling of the IRP and before ib_query calback*/
+       ref_al_obj( &h_qp->obj );               /* take IRP SA reference */
+
        /* query SA */
        status = ib_query( qp_get_al( h_qp ), &query_req, &h_qp->p_irp_que->h_query );
        if( status != IB_SUCCESS )
        {
                AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("ib_query failed (%d)\n", status) );
-               __ndi_complete_irp_ex( h_qp, CL_ERROR, NDI_CM_IDLE );
-               cl_status = CL_COMPLETED;
+               deref_al_obj( &h_qp->obj );     /* release IRP SA reference */
+               h_ioctl = IoCsqRemoveNextIrp( &h_qp->p_irp_que->csq, NULL );
+               if ( h_ioctl )
+               { /* IOCTL should be released in the caller */
+                       deref_al_obj( &h_qp->obj );             /* release IRP life reference */
+                       nt_status = STATUS_DRIVER_INTERNAL_ERROR;
+               }
        }
 
        AL_EXIT( AL_DBG_NDI );
-       return cl_status;
+       return nt_status;
 }
 
 NTSTATUS
@@ -1149,8 +1174,8 @@ ndi_req_cm(
        if ( h_qp->p_irp_que->h_ioctl )
        {
                AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, 
-                       ("STATUS_CONNECTION_ACTIVE: h_qp %I64x, ref_cnt %d\n",
-                       (uint64_t)h_qp, h_qp->obj.ref_cnt ) );
+                       ("STATUS_CONNECTION_ACTIVE: h_qp %#I64x, uhdl %#I64x, ref_cnt %d\n",
+                       (uint64_t)h_qp, h_qp->obj.hdl, h_qp->obj.ref_cnt ) );
                nt_status = STATUS_CONNECTION_ACTIVE;
                __ndi_release_lock( &h_qp->p_irp_que->csq, 0 );
                goto exit;
@@ -1172,8 +1197,15 @@ ndi_req_cm(
        nt_status = __ndi_ats_query( h_qp, h_ioctl, p_req );
 
 exit:
+       if ( nt_status != STATUS_PENDING )
+       {
+               cl_free( p_req );
+               h_qp->p_irp_que->state = NDI_CM_IDLE; 
+               h_qp->p_irp_que->h_ioctl = NULL; /* mark IRP as present */
+               cl_ioctl_complete( h_ioctl, nt_status, 0 );
+       }
        AL_EXIT( AL_DBG_NDI );
-       return nt_status;
+       return STATUS_PENDING;
 }
 
 
@@ -1232,14 +1264,13 @@ err:
                if( cid != AL_INVALID_CID )
                {
                        ref_al_obj( &h_qp->obj );
-                       if( al_destroy_cep( qp_get_al( h_qp ), 
-                               ((al_conn_qp_t*)h_qp)->cid, deref_al_obj ) != IB_SUCCESS )
+                       if( al_destroy_cep( qp_get_al( h_qp ), cid, deref_al_obj ) != IB_SUCCESS )
                        {
                                deref_al_obj( &h_qp->obj );
                        }
                }
 
-               AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
+               AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
                        ("al_cep_rtu returned %s.\n", ib_get_err_str( status )) );
                h_qp->p_irp_que->state = NDI_CM_IDLE;   
                nt_status = STATUS_DRIVER_INTERNAL_ERROR;
@@ -1252,9 +1283,12 @@ err:
        nt_status = STATUS_SUCCESS;
 
 exit:  
+       /* release the reference only for async case */
+       if ( p_irp->Tail.Overlay.DriverContext[1] )
+               deref_al_obj( &h_qp->obj );
+
        p_irp->IoStatus.Status = nt_status;
        IoCompleteRequest( p_irp, IO_NO_INCREMENT );
-       deref_al_obj( &h_qp->obj );
        AL_EXIT( AL_DBG_NDI );
 }
 
@@ -1273,6 +1307,8 @@ ndi_rtu_cm(
        p_irp->Tail.Overlay.DriverContext[1] = IoAllocateWorkItem( p_io_stack->DeviceObject );
        if ( p_irp->Tail.Overlay.DriverContext[1] )
        { /* asyncronous performing */
+               /* take a ref to prevent QP destroy before calling work item */
+               ref_al_obj( &h_qp->obj );
                IoMarkIrpPending( p_irp );
                IoQueueWorkItem( p_irp->Tail.Overlay.DriverContext[1],
                        __ndi_rtu_cm, DelayedWorkQueue, p_irp );
@@ -1358,9 +1394,12 @@ err:
        nt_status = STATUS_SUCCESS;
 
 exit:  
+       /* release the reference only for async case */
+       if ( p_irp->Tail.Overlay.DriverContext[1] )
+               deref_al_obj( &h_qp->obj );
+
        p_irp->IoStatus.Status = nt_status;
        IoCompleteRequest( p_irp, IO_NO_INCREMENT );
-       deref_al_obj( &h_qp->obj );
        AL_EXIT( AL_DBG_NDI );
 }
 
@@ -1411,7 +1450,7 @@ ndi_rep_cm(
        /* Format ib_cm_req_t structure */
        __ndi_fill_cm_rep( h_qp, p_rep, &cm_rep );
 
-       /* prepare CEP for connection */
+       /* prepare Passive CEP for connection */
        status = al_cep_pre_rep_ex( qp_get_al( h_qp ), cid, __ndi_cm_handler, h_qp, &cm_rep, &qp_mod );
        if( status != IB_SUCCESS )
        {
@@ -1432,6 +1471,10 @@ ndi_rep_cm(
                goto err_cep_destroy;
        }
 
+       AL_PRINT( TRACE_LEVEL_INFORMATION, AL_DBG_NDI,
+               ("Prepared Passive CEP with cid %d, h_al %p, context %p\n",
+               cid, qp_get_al( h_qp ), h_qp ) );
+
        /* See if this QP has already been connected. */
        old_cid = cl_atomic_comp_xchg( &((al_conn_qp_t*)h_qp)->cid, AL_INVALID_CID, cid );
        if( old_cid != AL_INVALID_CID )
@@ -1446,6 +1489,8 @@ ndi_rep_cm(
        p_irp->Tail.Overlay.DriverContext[1] = IoAllocateWorkItem( p_io_stack->DeviceObject );
        if ( p_irp->Tail.Overlay.DriverContext[1] )
        { /* asyncronous performing */
+               /* take a ref to prevent QP destroy before calling work item */
+               ref_al_obj( &h_qp->obj );
                IoMarkIrpPending( p_irp );
                IoQueueWorkItem( p_irp->Tail.Overlay.DriverContext[1],
                        __ndi_rep_cm, DelayedWorkQueue, p_irp );
@@ -1460,7 +1505,7 @@ err_cep_destroy:
        al_destroy_cep( qp_get_al( h_qp ), cid, NULL );
        
        h_qp->p_irp_que->state = NDI_CM_IDLE;   
-       __ndi_complete_irp( h_qp, p_irp, nt_status );
+       cl_ioctl_complete( p_irp, nt_status, 0 );
 
 exit:
        AL_EXIT( AL_DBG_NDI );
@@ -1519,8 +1564,7 @@ ndi_dreq_cm(
        if( cid != AL_INVALID_CID )
        {
                ref_al_obj( &h_qp->obj );
-               if( al_destroy_cep( qp_get_al( h_qp ), 
-                       ((al_conn_qp_t*)h_qp)->cid, deref_al_obj ) != IB_SUCCESS )
+               if( al_destroy_cep( qp_get_al( h_qp ), cid, deref_al_obj ) != IB_SUCCESS )
                {
                        deref_al_obj( &h_qp->obj );
                }
index 07b925637b5348998e5e47999ab90b36db031750..787e2e846ef2b6d64240c010f9929dda20c183c1 100644 (file)
@@ -135,8 +135,8 @@ ndi_qp_init(
 void
 ndi_qp_destroy(
        IN              ib_qp_handle_t                                  h_qp );
-void
 
+void
 ndi_qp_free(
        IN              ib_qp_handle_t                                  h_qp );
 
index e52e6094d4ee1a4a36baf13a2788ef2c59890cd9..67d1f9af7375711e6bc4ee9ccdcbdf300d23f13f 100644 (file)
@@ -877,7 +877,7 @@ proxy_cep_get_req_cid(
                cl_ioctl_out_size( h_ioctl ) != sizeof(uint32_t) )\r
        {\r
                AL_EXIT( AL_DBG_CM );\r
-               return CL_INVALID_REQUEST;\r
+               return CL_INVALID_PARAMETER;\r
        }\r
 \r
        /* get CID */\r
index a34ab9eab1ca4bd12a3bb6bc0845d53a51da833b..0c275b79c7104b3ba29490ac5cf5af0d9655c1e2 100644 (file)
@@ -86,7 +86,7 @@ __ndi_create_cq(
                cl_ioctl_in_size( h_ioctl ) != sizeof(p_ioctl->in) ||
                cl_ioctl_out_size( h_ioctl ) != sizeof(p_ioctl->out) )
        {
-               status = CL_INVALID_REQUEST;
+               status = CL_INVALID_PARAMETER;
                goto exit;
        }
 
@@ -170,7 +170,7 @@ __ndi_notify_cq(
        /* Validate user parameters. */
        if( cl_ioctl_in_size( h_ioctl ) != sizeof(ual_ndi_notify_cq_ioctl_in_t) )
        {
-               cl_status = CL_INVALID_REQUEST;
+               cl_status = CL_INVALID_PARAMETER;
                goto exit;
        }
 
@@ -214,7 +214,7 @@ __ndi_cancel_cq(
        /* Validate user parameters. */
        if( cl_ioctl_in_size( h_ioctl ) != sizeof(uint64_t) )
        {
-               cl_status = CL_INVALID_REQUEST;
+               cl_status = CL_INVALID_PARAMETER;
                goto exit;
        }
 
@@ -261,7 +261,7 @@ __ndi_modify_qp(
        /* Validate user parameters. */
        if( cl_ioctl_in_size( h_ioctl ) < sizeof(ual_ndi_modify_qp_ioctl_in_t))
        {
-               cl_status = CL_INVALID_REQUEST;
+               cl_status = CL_INVALID_PARAMETER;
                goto exit;
        }
 
@@ -323,7 +323,7 @@ __ndi_req_cm(
        /* Validate user parameters. */
        if( cl_ioctl_in_size( h_ioctl ) < sizeof(ual_ndi_req_cm_ioctl_in_t))
        {
-               cl_status = CL_INVALID_REQUEST;
+               cl_status = CL_INVALID_PARAMETER;
                goto exit;
        }
 
@@ -363,7 +363,7 @@ __ndi_req_cm(
        cl_status = ndi_req_cm( h_qp, h_ioctl, p_parm );
 
 exit:
-       if (h_qp && cl_status != CL_PENDING)
+       if (h_qp)
                deref_al_obj( &h_qp->obj );
 
        AL_EXIT( AL_DBG_NDI );
@@ -392,7 +392,7 @@ __ndi_rep_cm(
        if( (cl_ioctl_in_size( h_ioctl ) < sizeof(ual_ndi_rep_cm_ioctl_in_t)) ||
                cl_ioctl_out_size( h_ioctl ) < sizeof(net32_t) )
        {
-               cl_status = CL_INVALID_REQUEST;
+               cl_status = CL_INVALID_PARAMETER;
                goto exit;
        }
 
@@ -424,7 +424,7 @@ __ndi_rep_cm(
        cl_status = ndi_rep_cm( h_qp, cid, h_ioctl, p_rep );
 
 exit:
-       if (h_qp && cl_status != CL_PENDING)
+       if (h_qp)
                deref_al_obj( &h_qp->obj );
 
        AL_EXIT( AL_DBG_NDI );
@@ -501,7 +501,7 @@ __ndi_rtu_cm(
        /* Validate user parameters. */
        if( cl_ioctl_in_size( h_ioctl ) < sizeof(ual_ndi_rtu_cm_ioctl_in_t))
        {
-               cl_status = CL_INVALID_REQUEST;
+               cl_status = CL_INVALID_PARAMETER;
                goto exit;
        }
 
@@ -524,7 +524,7 @@ __ndi_rtu_cm(
        cl_status = ndi_rtu_cm( h_qp, h_ioctl );
 
 exit:
-       if (h_qp && cl_status != CL_PENDING)
+       if (h_qp)
                deref_al_obj( &h_qp->obj );
 
        AL_EXIT( AL_DBG_NDI );
@@ -550,7 +550,7 @@ __ndi_dreq_cm(
        /* Validate user parameters. */
        if( cl_ioctl_in_size( h_ioctl ) < sizeof(uint64_t))
        {
-               cl_status = CL_INVALID_REQUEST;
+               cl_status = CL_INVALID_PARAMETER;
                goto exit;
        }