From 545fc211aa6b22f3db2d99cff17efb1e4b78f289 Mon Sep 17 00:00:00 2001 From: leonidk Date: Sun, 27 Apr 2008 15:02:34 +0000 Subject: [PATCH] Synchronize cache work threads with its destruction git-svn-id: svn://openib.tc.cornell.edu/gen1@1077 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86 --- branches/ConnectX/hw/mlx4/kernel/core/cache.c | 49 +++++++----- .../ConnectX/hw/mlx4/kernel/inc/ib_verbs.h | 1 + .../ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h | 8 ++ .../ConnectX/hw/mlx4/kernel/inc/shutter.h | 78 +++++++++++++++++++ 4 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 branches/ConnectX/hw/mlx4/kernel/inc/shutter.h diff --git a/branches/ConnectX/hw/mlx4/kernel/core/cache.c b/branches/ConnectX/hw/mlx4/kernel/core/cache.c index 3d8112bb..2e281071 100644 --- a/branches/ConnectX/hw/mlx4/kernel/core/cache.c +++ b/branches/ConnectX/hw/mlx4/kernel/core/cache.c @@ -303,6 +303,7 @@ static void ib_work_item ( struct ib_update_work *work = (struct ib_update_work *)Context; UNREFERENCED_PARAMETER(DeviceObject); ib_cache_task(Context); + shutter_loose( &work->device->cache.x.work_thread ); IoFreeWorkItem(work->work_item); kfree(Context); } @@ -318,29 +319,29 @@ static void ib_cache_event(struct ib_event_handler *handler, event->event == IB_EVENT_PKEY_CHANGE || event->event == IB_EVENT_SM_CHANGE || event->event == IB_EVENT_CLIENT_REREGISTER) { + work = kmalloc(sizeof *work, GFP_ATOMIC); if (work) { - work->device = event->device; - work->port_num = event->element.port_num; + PDEVICE_OBJECT pdo = to_mdev(handler->device)->dev->pdev->p_self_do; + + // allocate work item + work->work_item = IoAllocateWorkItem(pdo); + if (work->work_item == NULL) { + //TODO: at least - print error. Need to return code, but the function is void + kfree( work ); + return; + } - { // schedule a work item to work - // get PDO - PDEVICE_OBJECT pdo = to_mdev(handler->device)->dev->pdev->p_self_do; - - // allocate work item - work->work_item = IoAllocateWorkItem(pdo); - if (work->work_item == NULL) { - //TODO: at least - print error. Need to return code, but the function is void - } - else { // schedule the work - IoQueueWorkItem( - work->work_item, - ib_work_item, - DelayedWorkQueue, - work - ); - } + // check whether we are in shutdown + if (shutter_use( &event->device->cache.x.work_thread ) <= 0) { + kfree( work ); + return; } + + // schedule the work + work->device = event->device; + work->port_num = event->element.port_num; + IoQueueWorkItem( work->work_item, ib_work_item, DelayedWorkQueue, work ); } } } @@ -370,6 +371,8 @@ static void ib_cache_setup_one(struct ib_device *device) goto err; } + shutter_init( &device->cache.x.work_thread ); + for (p = 0; p <= end_port(device) - start_port(device); ++p) { device->cache.pkey_cache[p] = NULL; device->cache.gid_cache [p] = NULL; @@ -395,13 +398,17 @@ err: kfree(device->cache.lmc_cache); } +// calling sequence: +// EvtReleaseHardware ==> mlx4_ib_cleanup ==> mlx4_unregister_interface ==> +// ==> mlx4_remove_device ==> mlx4_ib_remove ==> ib_unregister_device ==> +// ==> ib_cache_cleanup static void ib_cache_cleanup_one(struct ib_device *device) { int p; ib_unregister_event_handler(&device->cache.event_handler); - // TODO: how to do that ? - // LINUX: flush_scheduled_work(); + // instead of Linux flush_scheduled_work(): wait for them to quit + shutter_shut( &device->cache.x.work_thread ); for (p = 0; p <= end_port(device) - start_port(device); ++p) { kfree(device->cache.pkey_cache[p]); diff --git a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h index ba03ae28..16b2d08b 100644 --- a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h +++ b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs.h @@ -814,6 +814,7 @@ struct ib_cache { struct ib_pkey_cache **pkey_cache; struct ib_gid_cache **gid_cache; u8 *lmc_cache; + struct ib_cache_ex x; }; struct ib_dma_mapping_ops { diff --git a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h index 6ce8bc74..7c364954 100644 --- a/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h +++ b/branches/ConnectX/hw/mlx4/kernel/inc/ib_verbs_ex.h @@ -32,6 +32,8 @@ #pragma once +#include "shutter.h" + typedef struct _FDO_DEVICE_DATA *PFDO_DEVICE_DATA; struct ib_cq; @@ -95,4 +97,10 @@ struct ib_event_ex uint64_t vendor_specific; }; +/* extension for ib_cache */ +struct ib_cache_ex +{ + shutter_t work_thread; +}; + diff --git a/branches/ConnectX/hw/mlx4/kernel/inc/shutter.h b/branches/ConnectX/hw/mlx4/kernel/inc/shutter.h new file mode 100644 index 00000000..a483f28b --- /dev/null +++ b/branches/ConnectX/hw/mlx4/kernel/inc/shutter.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2005 SilverStorm Technologies. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id: shutter.h 1611 2006-08-20 14:48:55Z sleybo $ + */ + + +#pragma once + + +// Define the max numbers of operations that can be simultaniously done +#define MAX_OPERATIONS 0x10000000 + +typedef struct _shutter_t { + long cnt; + KEVENT event; + +} shutter_t; + +static inline void shutter_init(shutter_t* p_shutter) +{ + p_shutter->cnt = 0; + KeInitializeEvent( &p_shutter->event, SynchronizationEvent, FALSE ); +} + +// if RC > 0, one can proceed +static inline int shutter_use(shutter_t * p_shutter) +{ + long res = InterlockedIncrement( &p_shutter->cnt ); + ASSERT(res <= MAX_OPERATIONS); + if (res <= 0 ) + InterlockedDecrement( &p_shutter->cnt ); // The object is in shutdown + return res; +} + +static inline void shutter_loose(shutter_t * p_shutter) +{ + long res = InterlockedDecrement( &p_shutter->cnt ); + if (res == -MAX_OPERATIONS) + KeSetEvent( &p_shutter->event, 0, FALSE ); +} + +static inline void shutter_shut(shutter_t * p_shutter) +{ + long res; + // Mark the counter as locked + res = InterlockedExchangeAdd(&p_shutter->cnt, -MAX_OPERATIONS); + ASSERT(res >= 0); + if (res) + // We are now waiting for the object to reach -MAX_OPERATIONS + KeWaitForSingleObject( &p_shutter->event, Executive, KernelMode, FALSE, NULL ); +} + -- 2.41.0