]> git.openfabrics.org - ~shefty/rdma-win.git/commitdiff
[IPoIB_ndis6_cm] migrate IPoIB shutter code into the trunk - match 2.2 release.
authorStan Smith <stan.smith@intel.com>
Fri, 19 Feb 2010 18:13:20 +0000 (18:13 +0000)
committerStan Smith <stan.smith@intel.com>
Fri, 19 Feb 2010 18:13:20 +0000 (18:13 +0000)
shutter code stops IPoIB from receiving packets during device shutdown.

git-svn-id: svn://openib.tc.cornell.edu/gen1@2710 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

trunk/hw/mlx4/kernel/inc/shutter.h [deleted file]
trunk/inc/kernel/shutter.h [new file with mode: 0644]
trunk/ulp/ipoib_NDIS6_CM/kernel/ipoib_adapter.cpp
trunk/ulp/ipoib_NDIS6_CM/kernel/ipoib_adapter.h
trunk/ulp/ipoib_NDIS6_CM/kernel/ipoib_driver.cpp
trunk/ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp

diff --git a/trunk/hw/mlx4/kernel/inc/shutter.h b/trunk/hw/mlx4/kernel/inc/shutter.h
deleted file mode 100644 (file)
index 2aed129..0000000
+++ /dev/null
@@ -1,119 +0,0 @@
-/*\r
- * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
- *\r
- * This software is available to you under the OpenIB.org BSD license\r
- * below:\r
- *\r
- *     Redistribution and use in source and binary forms, with or\r
- *     without modification, are permitted provided that the following\r
- *     conditions are met:\r
- *\r
- *      - Redistributions of source code must retain the above\r
- *        copyright notice, this list of conditions and the following\r
- *        disclaimer.\r
- *\r
- *      - Redistributions in binary form must reproduce the above\r
- *        copyright notice, this list of conditions and the following\r
- *        disclaimer in the documentation and/or other materials\r
- *        provided with the distribution.\r
- *\r
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
- * SOFTWARE.\r
- *\r
- * $Id: shutter.h 1611 2006-08-20 14:48:55Z sleybo $\r
- */\r
-\r
-\r
-#pragma once\r
-\r
-\r
-// Define the max numbers of operations that can be simultaniously done\r
-#define MAX_OPERATIONS 0x10000000\r
-\r
-typedef struct _shutter_t {\r
-       long cnt;\r
-       KEVENT event;\r
-\r
-}      shutter_t;\r
-\r
-static inline void shutter_init(shutter_t* p_shutter)\r
-{\r
-       p_shutter->cnt = 0;\r
-       KeInitializeEvent( &p_shutter->event, SynchronizationEvent, FALSE );\r
-}\r
-\r
-\r
-static inline void shutter_sub(shutter_t * p_shutter,long Val)\r
-{\r
-    long res = 0;\r
-    ASSERT(Val < 0);\r
-       res = InterlockedExchangeAdd( &p_shutter->cnt,Val );\r
-       if ((res+Val) == -MAX_OPERATIONS)\r
-               KeSetEvent( &p_shutter->event, 0, FALSE );\r
-}\r
-\r
-// if RC == true, one can proceed\r
-static inline BOOLEAN shutter_add(shutter_t * p_shutter,long Val)\r
-{\r
-    long res = 0;\r
-    ASSERT(Val > 0);\r
-       res = InterlockedExchangeAdd(&p_shutter->cnt,Val); \r
-       ASSERT(res <= MAX_OPERATIONS);\r
-       if (res < 0 )\r
-       {        \r
-               shutter_sub(p_shutter, -Val);\r
-               return FALSE;\r
-       }\r
-       return TRUE;\r
-}\r
-\r
-static inline void shutter_loose(shutter_t * p_shutter)\r
-{\r
-       long res = InterlockedDecrement( &p_shutter->cnt );\r
-       if (res == -MAX_OPERATIONS)\r
-               KeSetEvent( &p_shutter->event, 0, FALSE );\r
-}\r
-\r
-// if RC > 0, one can proceed\r
-static inline int shutter_use(shutter_t * p_shutter)\r
-{\r
-       long res = InterlockedIncrement( &p_shutter->cnt ); \r
-       ASSERT(res <= MAX_OPERATIONS);\r
-       if (res <= 0 ) \r
-               shutter_loose( p_shutter ); // The object is in shutdown\r
-       return res;\r
-}\r
-\r
-\r
-static inline void shutter_shut(shutter_t * p_shutter)\r
-{\r
-    long res = 0;\r
-    //\r
-    //  ASSERT not calling shu twice.\r
-    //\r
-    ASSERT(p_shutter->cnt - MAX_OPERATIONS >=  (-MAX_OPERATIONS));\r
-    \r
-       // Mark the counter as locked\r
-       res = InterlockedExchangeAdd(&p_shutter->cnt, -MAX_OPERATIONS);\r
-       ASSERT(res >= 0);\r
-       if (res) \r
-               // We are now waiting for the object to reach -MAX_OPERATIONS\r
-               KeWaitForSingleObject( &p_shutter->event, Executive, KernelMode, FALSE, NULL );\r
-}\r
-\r
-static inline void shutter_alive(shutter_t * p_shutter)\r
-{\r
-    long res = 0;\r
-    \r
-       // Mark the counter as alive\r
-       res = InterlockedExchangeAdd(&p_shutter->cnt, MAX_OPERATIONS);\r
-       ASSERT(res < 0);\r
-}\r
-\r
-\r
diff --git a/trunk/inc/kernel/shutter.h b/trunk/inc/kernel/shutter.h
new file mode 100644 (file)
index 0000000..2aed129
--- /dev/null
@@ -0,0 +1,119 @@
+/*\r
+ * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id: shutter.h 1611 2006-08-20 14:48:55Z sleybo $\r
+ */\r
+\r
+\r
+#pragma once\r
+\r
+\r
+// Define the max numbers of operations that can be simultaniously done\r
+#define MAX_OPERATIONS 0x10000000\r
+\r
+typedef struct _shutter_t {\r
+       long cnt;\r
+       KEVENT event;\r
+\r
+}      shutter_t;\r
+\r
+static inline void shutter_init(shutter_t* p_shutter)\r
+{\r
+       p_shutter->cnt = 0;\r
+       KeInitializeEvent( &p_shutter->event, SynchronizationEvent, FALSE );\r
+}\r
+\r
+\r
+static inline void shutter_sub(shutter_t * p_shutter,long Val)\r
+{\r
+    long res = 0;\r
+    ASSERT(Val < 0);\r
+       res = InterlockedExchangeAdd( &p_shutter->cnt,Val );\r
+       if ((res+Val) == -MAX_OPERATIONS)\r
+               KeSetEvent( &p_shutter->event, 0, FALSE );\r
+}\r
+\r
+// if RC == true, one can proceed\r
+static inline BOOLEAN shutter_add(shutter_t * p_shutter,long Val)\r
+{\r
+    long res = 0;\r
+    ASSERT(Val > 0);\r
+       res = InterlockedExchangeAdd(&p_shutter->cnt,Val); \r
+       ASSERT(res <= MAX_OPERATIONS);\r
+       if (res < 0 )\r
+       {        \r
+               shutter_sub(p_shutter, -Val);\r
+               return FALSE;\r
+       }\r
+       return TRUE;\r
+}\r
+\r
+static inline void shutter_loose(shutter_t * p_shutter)\r
+{\r
+       long res = InterlockedDecrement( &p_shutter->cnt );\r
+       if (res == -MAX_OPERATIONS)\r
+               KeSetEvent( &p_shutter->event, 0, FALSE );\r
+}\r
+\r
+// if RC > 0, one can proceed\r
+static inline int shutter_use(shutter_t * p_shutter)\r
+{\r
+       long res = InterlockedIncrement( &p_shutter->cnt ); \r
+       ASSERT(res <= MAX_OPERATIONS);\r
+       if (res <= 0 ) \r
+               shutter_loose( p_shutter ); // The object is in shutdown\r
+       return res;\r
+}\r
+\r
+\r
+static inline void shutter_shut(shutter_t * p_shutter)\r
+{\r
+    long res = 0;\r
+    //\r
+    //  ASSERT not calling shu twice.\r
+    //\r
+    ASSERT(p_shutter->cnt - MAX_OPERATIONS >=  (-MAX_OPERATIONS));\r
+    \r
+       // Mark the counter as locked\r
+       res = InterlockedExchangeAdd(&p_shutter->cnt, -MAX_OPERATIONS);\r
+       ASSERT(res >= 0);\r
+       if (res) \r
+               // We are now waiting for the object to reach -MAX_OPERATIONS\r
+               KeWaitForSingleObject( &p_shutter->event, Executive, KernelMode, FALSE, NULL );\r
+}\r
+\r
+static inline void shutter_alive(shutter_t * p_shutter)\r
+{\r
+    long res = 0;\r
+    \r
+       // Mark the counter as alive\r
+       res = InterlockedExchangeAdd(&p_shutter->cnt, MAX_OPERATIONS);\r
+       ASSERT(res < 0);\r
+}\r
+\r
+\r
index c18dfee7736ee2301a9b87511b79d903c2ecdcd9..769739693e4b2b9aa0b9bc5790b3e3def5fb99a4 100644 (file)
@@ -390,6 +390,13 @@ adapter_init(
                        p_adapter->p_ifc->get_err_str( status )) );\r
                return status;\r
        }\r
+       \r
+       \r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+               ("Shutter Init, state = %d\n", p_adapter->ipoib_state) );\r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+               ("Shutter Init, state = %d\n", p_adapter->ipoib_state) );\r
+       shutter_init( &p_adapter->recv_shutter );\r
 \r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return status;\r
index c0718e9af00adefdefe484a993cbd380d1e233d6..de28d977af00e62ed172271ef0ef82cb18eab36a 100644 (file)
@@ -47,6 +47,8 @@
 #include <ip_packet.h>\r
 #include "ip_stats.h"\r
 #include "ipoib_stat.h"\r
+#include "shutter.h"\r
+\r
 \r
 \r
 /*\r
@@ -70,6 +72,7 @@ typedef enum
 \r
 typedef enum _ipoib_state\r
 {\r
+       IPOIB_INIT = -1,\r
     IPOIB_PAUSED,\r
     IPOIB_PAUSING,\r
     IPOIB_RUNNING\r
@@ -238,6 +241,9 @@ typedef struct _ipoib_adapter
        ULONG                                   n_send_NBL;                     // number of send NBLs, gotten from NDIS\r
        ULONG                                   n_send_NBL_done;        // number of send NBLs, completed\r
 \r
+       //\r
+       shutter_t               recv_shutter;\r
+\r
 }      ipoib_adapter_t;\r
 /*\r
 * FIELDS\r
index fff1d6cc7031dece8a0ac574ba97241981663f90..cbe6cd9dd7fe2c9270ea636bc2ced9cc473892d9 100644 (file)
@@ -1862,7 +1862,7 @@ if(cl_get_time_stamp_sec() < 30) {
                                ("ipoib_create_adapter returned status %d.\n", ib_status ) );\r
                        return NDIS_STATUS_FAILURE;\r
                }\r
-               p_adapter->ipoib_state = IPOIB_PAUSED;\r
+               p_adapter->ipoib_state = IPOIB_INIT;\r
 \r
                status  = SetAttributes(p_adapter, h_adapter);\r
                if (status != NDIS_STATUS_SUCCESS) {\r
@@ -3108,13 +3108,12 @@ ipoib_send_net_buffer_list(
 \r
        CL_ASSERT( adapter_context );\r
        p_adapter = (ipoib_adapter_t*)adapter_context;\r
-       p_port = p_adapter->p_port;\r
 \r
        cl_obj_lock( &p_adapter->obj );\r
        if( p_adapter->ipoib_state == IPOIB_PAUSING ||\r
                p_adapter->ipoib_state == IPOIB_PAUSED)\r
        {\r
-               status = NDIS_STATUS_PAUSED; //NDIS_STATUS_PAUSED; \r
+               status = NDIS_STATUS_PAUSED; \r
                IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
                                ("Got send during PAUSE, complete with error \n") );\r
                cl_obj_unlock( &p_adapter->obj );\r
@@ -3236,9 +3235,21 @@ ipoib_shutdown_ex(
        IN NDIS_HANDLE  adapter_context,\r
        IN NDIS_SHUTDOWN_ACTION  shutdown_action)\r
 {\r
-       IPOIB_ENTER( IPOIB_DBG_INIT );\r
-       UNUSED_PARAM( adapter_context );\r
+       IPOIB_ENTER( IPOIB_DBG_INIT ) ;\r
        UNUSED_PARAM( shutdown_action );\r
+       \r
+       ipoib_adapter_t *p_adapter = (ipoib_adapter_t *) adapter_context;\r
+       \r
+       if (shutdown_action == NdisShutdownPowerOff ) {\r
+               ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);\r
+               // We need to wait only if this is not a blue screen any way\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,\r
+                       ("Shutter shut, state = %d\n", p_adapter->ipoib_state));\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+                       ("Shutter shut, state = %d\n", p_adapter->ipoib_state));\r
+               shutter_shut ( &p_adapter->recv_shutter );\r
+       }\r
+\r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
 }\r
 \r
@@ -3938,13 +3949,15 @@ ipoib_pause(
        KeReleaseInStackQueuedSpinLock( &hdl );\r
        \r
        if (p_adapter->p_port) {\r
-               //TODO improve this flow !\r
-               //TODO Be sure we stopped all sends and receives\r
                cl_spinlock_acquire( &p_adapter->p_port->send_lock );\r
                ipoib_port_resume(p_adapter->p_port,FALSE);\r
                cl_spinlock_release( &p_adapter->p_port->send_lock );\r
        }\r
-\r
+       \r
+       IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+               ("Shutter shut, state = %d\n", p_adapter->ipoib_state));\r
+       shutter_shut ( &p_adapter->recv_shutter );\r
+       \r
        KeAcquireInStackQueuedSpinLock( &g_ipoib.lock, &hdl );\r
        p_adapter->ipoib_state = IPOIB_PAUSED;\r
        KeReleaseInStackQueuedSpinLock( &hdl );\r
@@ -3975,6 +3988,16 @@ ipoib_restart(
         //\r
         // Check to see if we need to change any attributes\r
     }\r
+       \r
+       if ( (p_adapter->ipoib_state == IPOIB_PAUSED) || (p_adapter->ipoib_state == IPOIB_INIT) ) {\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+                       ("Shutter Alive, ipoib_state = %d\n", p_adapter->ipoib_state));\r
+               shutter_alive( &p_adapter->recv_shutter );\r
+       }\r
+       else {\r
+               IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_RECV,\r
+               ("*****Shutter Was not \"Alived\", state = %d*****\n", p_adapter->ipoib_state));\r
+       }\r
 \r
        KeAcquireInStackQueuedSpinLock( &g_ipoib.lock, &hdl );\r
        p_adapter->ipoib_state = IPOIB_RUNNING;\r
index df8d9bf2dc8d3463af86b6d4ffafa8ec0b624f5a..a5c5aaca1179913ff1c0c05acebea398e8709430 100644 (file)
@@ -824,7 +824,21 @@ __port_init(
        IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
                ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
 #endif\r
-\r
+    // The port is started as paused and NDIS calls latter to ipoib_restart. We\r
+    // shut the recv_shuter for now and alive it on ipoib_restart. We did\r
+    // it in this way since MpInitializeInternal also calls in reset and than\r
+    // we need to set the rec ref count to 1 //TODO !!!!!!!!!!1\r
+    //\r
+    \r
+       if ( p_adapter->ipoib_state == IPOIB_INIT) {\r
+               IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
+               ("Shutter shut, state = %d\n", p_adapter->ipoib_state));\r
+               shutter_shut ( &p_adapter->recv_shutter );\r
+       }\r
+       else {\r
+               IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_RECV,\r
+               ("*****Shutter wasn't shut, state = %d*****\n", p_adapter->ipoib_state));\r
+       }\r
        IPOIB_EXIT( IPOIB_DBG_INIT );\r
        return IB_SUCCESS;\r
 }\r
@@ -1843,44 +1857,27 @@ __recv_mgr_repost(
        return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
 }\r
 \r
-void\r
-ipoib_return_net_buffer_list(\r
-       IN                              NDIS_HANDLE                                     adapter_context,\r
-       IN                              NET_BUFFER_LIST                         *p_net_buffer_lists,\r
-       IN                              ULONG                                           return_flags)\r
+inline ULONG __free_received_NBL (\r
+       IN ipoib_port_t         *p_port,\r
+       IN NET_BUFFER_LIST      *p_net_buffer_lists\r
+       ) \r
 {\r
-       ipoib_port_t            *p_port;\r
-       ipoib_recv_desc_t       *p_desc;\r
-       NET_BUFFER_LIST         *cur_net_buffer_list,*next_net_buffer_list;\r
-       int32_t                         shortage;\r
-       \r
-       PERF_DECLARE( ReturnPacket );\r
-       PERF_DECLARE( ReturnPutRecv );\r
-       PERF_DECLARE( ReturnRepostRecv );\r
-       PERF_DECLARE( ReturnPreparePkt );\r
-       PERF_DECLARE( ReturnNdisIndicate );\r
 \r
-       IPOIB_ENTER( IPOIB_DBG_RECV );\r
-\r
-       UNUSED_PARAM( return_flags );\r
-\r
-       p_port = ((ipoib_adapter_t*)adapter_context)->p_port;\r
-       CL_ASSERT( p_net_buffer_lists );\r
+       ipoib_recv_desc_t       *p_desc;\r
+       NET_BUFFER_LIST         *cur_net_buffer_list, *next_net_buffer_list;\r
+       LONG                            NBL_cnt = 0;\r
 \r
-       cl_perf_start( ReturnPacket );\r
-       cl_spinlock_acquire( &p_port->recv_lock );\r
+       \r
        for (cur_net_buffer_list = p_net_buffer_lists;\r
                 cur_net_buffer_list != NULL;\r
                 cur_net_buffer_list = next_net_buffer_list)\r
        {\r
+               ++NBL_cnt;\r
                next_net_buffer_list = NET_BUFFER_LIST_NEXT_NBL(cur_net_buffer_list);\r
 \r
                /* Get the port and descriptor from the NET_BUFFER_LIST. */\r
                CL_ASSERT(p_port == IPOIB_PORT_FROM_NBL( cur_net_buffer_list ));\r
                p_desc = IPOIB_RECV_FROM_NBL( cur_net_buffer_list );\r
-\r
-               \r
-               //TODO: NDIS60, rewrite this block\r
                \r
 #if 0 //TODO CM flow\r
                if( p_desc->type == PKT_TYPE_CM_UCAST )\r
@@ -1916,8 +1913,43 @@ ipoib_return_net_buffer_list(
                __buf_mgr_put_recv( p_port, p_desc, cur_net_buffer_list );\r
                cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
        }\r
+       return NBL_cnt;\r
+}\r
 \r
+void\r
+ipoib_return_net_buffer_list(\r
+       IN                              NDIS_HANDLE                                     adapter_context,\r
+       IN                              NET_BUFFER_LIST                         *p_net_buffer_lists,\r
+       IN                              ULONG                                           return_flags)\r
+{\r
+       ipoib_port_t            *p_port;\r
+       int32_t                         shortage;\r
+       LONG                            NBL_cnt = 0;\r
+       \r
+       PERF_DECLARE( ReturnPacket );\r
+       PERF_DECLARE( ReturnPutRecv );\r
+       PERF_DECLARE( ReturnRepostRecv );\r
+       PERF_DECLARE( ReturnPreparePkt );\r
+       PERF_DECLARE( ReturnNdisIndicate );\r
+\r
+       IPOIB_ENTER( IPOIB_DBG_RECV );\r
+\r
+       UNUSED_PARAM( return_flags );\r
+       \r
+       p_port = ((ipoib_adapter_t*)adapter_context)->p_port;\r
+       CL_ASSERT( p_net_buffer_lists );\r
+       if ( !p_port ) {\r
+               ASSERT(p_port);\r
+               IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
+                       ("return_NBL callback called when port pointer was already cleared\n") );\r
+               return;\r
+       }\r
+\r
+       cl_perf_start( ReturnPacket );\r
+       cl_spinlock_acquire( &p_port->recv_lock );\r
+       NBL_cnt = __free_received_NBL( p_port, p_net_buffer_lists );\r
 \r
+       shutter_sub( &p_port->p_adapter->recv_shutter, -NBL_cnt );\r
 \r
        /* Repost buffers to HW */\r
        cl_perf_start( ReturnRepostRecv );\r
@@ -1958,6 +1990,7 @@ __recv_cb(
        cl_qlist_t                      done_list, bad_list;\r
        size_t                          i;\r
        ULONG                           recv_complete_flags = 0;\r
+       BOOLEAN                         res;\r
 \r
        PERF_DECLARE( RecvCompBundle );\r
        PERF_DECLARE( RecvCb );\r
@@ -2069,12 +2102,39 @@ __recv_cb(
 \r
                cl_perf_start( RecvNdisIndicate );\r
                \r
-               NdisMIndicateReceiveNetBufferLists(\r
-                       p_port->p_adapter->h_adapter,\r
-                       p_port->recv_mgr.recv_NBL_array[0],\r
-                       NDIS_DEFAULT_PORT_NUMBER,\r
-                       NBL_cnt,\r
-                       recv_complete_flags);\r
+               if (shortage <= 0) {\r
+\r
+                       res = shutter_add( &p_port->p_adapter->recv_shutter, NBL_cnt );\r
+                       if (res) {\r
+                               NdisMIndicateReceiveNetBufferLists(\r
+                                       p_port->p_adapter->h_adapter,\r
+                                       p_port->recv_mgr.recv_NBL_array[0],\r
+                                       NDIS_DEFAULT_PORT_NUMBER,\r
+                                       NBL_cnt,\r
+                                       recv_complete_flags);\r
+                       }\r
+                       else {\r
+                               __free_received_NBL (p_port, p_port->recv_mgr.recv_NBL_array[0]);\r
+                       }\r
+                               \r
+               } else {\r
+               \r
+                       // If shortage >0, we already set the status to NDIS_RECEIVE_FLAGS_RESOURCES\r
+                       // That is, IPoIB driver regain ownership of the NET_BUFFER_LIST structures immediately\r
+                       res = shutter_add( &p_port->p_adapter->recv_shutter, 1 );\r
+                       if (res) {\r
+                               NdisMIndicateReceiveNetBufferLists(\r
+                                       p_port->p_adapter->h_adapter,\r
+                                       p_port->recv_mgr.recv_NBL_array[0],\r
+                                       NDIS_DEFAULT_PORT_NUMBER,\r
+                                       NBL_cnt,\r
+                                       recv_complete_flags);\r
+                               shutter_sub( &p_port->p_adapter->recv_shutter, -1 );\r
+                       }\r
+                       \r
+               }\r
+               \r
+       \r
 \r
                cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
 \r
@@ -3243,14 +3303,8 @@ __pending_list_destroy(
 {\r
        cl_list_item_t          *p_item;\r
        ipoib_send_NB_SG        *s_buf;\r
-       ULONG                           send_complete_flags = 0;\r
-\r
-       if (KeGetCurrentIrql() == DISPATCH_LEVEL)\r
-       {\r
-               NDIS_SET_SEND_COMPLETE_FLAG(send_complete_flags, NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL);\r
-       } \r
+       ULONG                           send_complete_flags = NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL;\r
 \r
-       cl_spinlock_acquire( &p_port->send_lock );\r
        /* Complete any pending packets. */\r
        for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
                p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
@@ -3260,13 +3314,10 @@ __pending_list_destroy(
                ASSERT(s_buf->p_port == p_port);\r
                ASSERT(s_buf->p_nbl);\r
 \r
-               \r
+               //TODO\r
                //__send_complete_net_buffer(s_buf, NDIS_STATUS_RESET_IN_PROGRESS,send_complete_flags,TRUE);\r
                __send_complete_net_buffer(s_buf, NDIS_STATUS_FAILURE,send_complete_flags,TRUE);\r
        }\r
-\r
-\r
-       cl_spinlock_release( &p_port->send_lock );\r
   \r
 }\r
 \r
@@ -3278,7 +3329,9 @@ __send_mgr_destroy(
        //Destroy pending list and put all the send buffers back to pool\r
        //The list should be already destroyed at this point\r
        ASSERT(p_port->send_mgr.pending_list.count == 0);\r
+       cl_spinlock_acquire( &p_port->send_lock );\r
        __pending_list_destroy(p_port);\r
+       cl_spinlock_release( &p_port->send_lock );\r
 \r
        // Now, destroy the send pool\r
        cl_qpool_destroy(&p_port->send_mgr.send_pool);\r
@@ -5298,7 +5351,6 @@ ipoib_port_send(
                //TODO Tzachid: make an assert here to validate your IRQL\r
                ASSERT (KeGetCurrentIrql() == DISPATCH_LEVEL);\r
                old_irql = DISPATCH_LEVEL;\r
-               NDIS_SET_SEND_COMPLETE_FLAG(send_complete_flags, NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL);\r
        } else {\r
                NDIS_RAISE_IRQL_TO_DISPATCH(&old_irql);\r
                //ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL); // Happens\r
@@ -7157,6 +7209,7 @@ ipoib_port_down(
         * object lock since that is the order taken when reposting.\r
         */\r
        cl_spinlock_acquire( &p_port->recv_lock );\r
+       cl_spinlock_acquire( &p_port->send_lock );\r
        cl_obj_lock( &p_port->obj );\r
        p_port->state = IB_QPS_ERROR;\r
 \r
@@ -7171,6 +7224,7 @@ ipoib_port_down(
                p_port->ib_mgr.h_query = NULL;\r
        }\r
        cl_obj_unlock( &p_port->obj );\r
+       cl_spinlock_release( &p_port->send_lock );\r
        cl_spinlock_release( &p_port->recv_lock );\r
 \r
        KeWaitForSingleObject(\r
@@ -7669,10 +7723,10 @@ __mcast_cb(
                IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_MCAST ,("Invalid state - Aborting.\n") );\r
                IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT ,("Invalid state - Aborting.\n") );\r
 \r
-               //cl_spinlock_acquire(&p_port->send_lock);\r
+               cl_spinlock_acquire(&p_port->send_lock);\r
                //ipoib_port_resume(p_port , FALSE);\r
                __pending_list_destroy( p_port );\r
-               //cl_spinlock_release(&p_port->send_lock);\r
+               cl_spinlock_release(&p_port->send_lock);\r
                return;\r
        }\r
 \r