Synchronize cache work threads with its destruction

git-svn-id: svn://openib.tc.cornell.edu/gen1@1077 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/branches/ConnectX/hw/mlx4/kernel/core/cache.c b/branches/ConnectX/hw/mlx4/kernel/core/cache.c
index 3d8112bbc7d0731b82bf72cd14c54ed42f87d04b..2e2810714a08d2723de1322ac6f2c5aac761fa72 100644 (file)
--- a/branches/ConnectX/hw/mlx4/kernel/core/cache.c
+++ b/branches/ConnectX/hw/mlx4/kernel/core/cache.c
@@ -303,6 +303,7 @@ static void ib_work_item (
        struct ib_update_work *work = (struct ib_update_work *)Context;
        UNREFERENCED_PARAMETER(DeviceObject);
        ib_cache_task(Context);
+       shutter_loose( &work->device->cache.x.work_thread );
        IoFreeWorkItem(work->work_item);
        kfree(Context);
 }
@@ -318,29 +319,29 @@ static void ib_cache_event(struct ib_event_handler *handler,
                event->event == IB_EVENT_PKEY_CHANGE ||
                event->event == IB_EVENT_SM_CHANGE   ||
                event->event == IB_EVENT_CLIENT_REREGISTER) {
+
                work = kmalloc(sizeof *work, GFP_ATOMIC);
                if (work) {
-                       work->device   = event->device;
-                       work->port_num = event->element.port_num;
+                       PDEVICE_OBJECT pdo = to_mdev(handler->device)->dev->pdev->p_self_do;
+
+                       // allocate work item
+                       work->work_item = IoAllocateWorkItem(pdo);
+                       if (work->work_item == NULL) {
+                               //TODO: at least - print error. Need to return code, but the function is void 
+                               kfree( work );
+                               return;
+                       }
 
-                       { // schedule a work item to work
-                               // get PDO
-                               PDEVICE_OBJECT pdo = to_mdev(handler->device)->dev->pdev->p_self_do;
-               
-                               // allocate work item
-                               work->work_item = IoAllocateWorkItem(pdo);
-                               if (work->work_item == NULL) {
-                                       //TODO: at least - print error. Need to return code, but the function is void 
-                               }
-                               else { // schedule the work
-                                       IoQueueWorkItem(
-                                                       work->work_item,
-                                                       ib_work_item,
-                                                       DelayedWorkQueue,
-                                                       work
-                                                       );
-                               }
+                       // check whether we are in shutdown
+                       if (shutter_use( &event->device->cache.x.work_thread ) <= 0) {
+                               kfree( work );
+                               return;
                        }
+
+                       // schedule the work
+                       work->device   = event->device;
+                       work->port_num = event->element.port_num;
+                       IoQueueWorkItem( work->work_item, ib_work_item, DelayedWorkQueue, work );
                }
        }
 }
@@ -370,6 +371,8 @@ static void ib_cache_setup_one(struct ib_device *device)
                goto err;
        }
 
+       shutter_init( &device->cache.x.work_thread );
+
        for (p = 0; p <= end_port(device) - start_port(device); ++p) {
                device->cache.pkey_cache[p] = NULL;
                device->cache.gid_cache [p] = NULL;
@@ -395,13 +398,17 @@ err:
        kfree(device->cache.lmc_cache);
 }
 
+// calling sequence:
+// EvtReleaseHardware ==> mlx4_ib_cleanup ==> mlx4_unregister_interface ==>
+// ==> mlx4_remove_device ==> mlx4_ib_remove ==> ib_unregister_device ==>
+// ==> ib_cache_cleanup
 static void ib_cache_cleanup_one(struct ib_device *device)
 {
        int p;
 
        ib_unregister_event_handler(&device->cache.event_handler);
-       // TODO: how to do that ?
-       // LINUX: flush_scheduled_work();
+       // instead of Linux flush_scheduled_work(): wait for them to quit
+       shutter_shut( &device->cache.x.work_thread );
 
        for (p = 0; p <= end_port(device) - start_port(device); ++p) {
                kfree(device->cache.pkey_cache[p]);

diff --git a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h
index ba03ae28546acd694dec6ca169da71ef25416040..16b2d08b189cf5e69c98d897e757a10244583a8a 100644 (file)
--- a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h
+++ b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h
@@ -814,6 +814,7 @@ struct ib_cache {
        struct ib_pkey_cache  **pkey_cache;
        struct ib_gid_cache   **gid_cache;
        u8                     *lmc_cache;
+       struct ib_cache_ex      x;
 };
 
 struct ib_dma_mapping_ops {

diff --git a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h
index 6ce8bc749e6cdfbbe284ac5c007f4a2137d950e9..7c364954a0370ed7a07e630e6c5100c59fd71b69 100644 (file)
--- a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h
+++ b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h
@@ -32,6 +32,8 @@
 
 #pragma once
 
+#include "shutter.h"
+
 typedef struct _FDO_DEVICE_DATA *PFDO_DEVICE_DATA;
 struct ib_cq;
 
@@ -95,4 +97,10 @@ struct ib_event_ex
        uint64_t                vendor_specific;
 };
 
+/* extension for ib_cache */
+struct ib_cache_ex 
+{
+       shutter_t               work_thread;
+};
+
 

diff --git a/branches/ConnectX/hw/mlx4/kernel/inc/shutter.h b/branches/ConnectX/hw/mlx4/kernel/inc/shutter.h
new file mode 100644 (file)
index 0000000..a483f28
--- /dev/null
+++ b/branches/ConnectX/hw/mlx4/kernel/inc/shutter.h
@@ -0,0 +1,78 @@
+/*\r
+ * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id: shutter.h 1611 2006-08-20 14:48:55Z sleybo $\r
+ */\r
+\r
+\r
+#pragma once\r
+\r
+\r
+// Define the max numbers of operations that can be simultaniously done\r
+#define MAX_OPERATIONS 0x10000000\r
+\r
+typedef struct _shutter_t {\r
+       long cnt;\r
+       KEVENT event;\r
+\r
+}      shutter_t;\r
+\r
+static inline void shutter_init(shutter_t* p_shutter)\r
+{\r
+       p_shutter->cnt = 0;\r
+       KeInitializeEvent( &p_shutter->event, SynchronizationEvent, FALSE );\r
+}\r
+\r
+// if RC > 0, one can proceed\r
+static inline int shutter_use(shutter_t * p_shutter)\r
+{\r
+       long res = InterlockedIncrement( &p_shutter->cnt ); \r
+       ASSERT(res <= MAX_OPERATIONS);\r
+       if (res <= 0 ) \r
+               InterlockedDecrement( &p_shutter->cnt ); // The object is in shutdown\r
+       return res;\r
+}\r
+\r
+static inline void shutter_loose(shutter_t * p_shutter)\r
+{\r
+       long res = InterlockedDecrement( &p_shutter->cnt );\r
+       if (res == -MAX_OPERATIONS)\r
+               KeSetEvent( &p_shutter->event, 0, FALSE );\r
+}\r
+\r
+static inline void shutter_shut(shutter_t * p_shutter)\r
+{\r
+       long res;\r
+       // Mark the counter as locked\r
+       res = InterlockedExchangeAdd(&p_shutter->cnt, -MAX_OPERATIONS);\r
+       ASSERT(res >= 0);\r
+       if (res) \r
+               // We are now waiting for the object to reach -MAX_OPERATIONS\r
+               KeWaitForSingleObject( &p_shutter->event, Executive, KernelMode, FALSE, NULL );\r
+}\r
+\r