[MTHCA] improve spinlocks to take a more efficient spinlock while at DPC level

git-svn-id: svn://openib.tc.cornell.edu/gen1@578 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/hw/mthca/kernel/mt_spinlock.h b/trunk/hw/mthca/kernel/mt_spinlock.h
index 9227365ace748c4b0bf9664ddfed41df817bbe07..111d02aa554ff36ecf4986064bcdde887788bffb 100644 (file)
--- a/trunk/hw/mthca/kernel/mt_spinlock.h
+++ b/trunk/hw/mthca/kernel/mt_spinlock.h
@@ -2,17 +2,23 @@
 #define MT_SPINLOCK_H
 
 typedef struct spinlock {
-       KSPIN_LOCK              lock;
-#ifdef SUPPORT_SPINLOCK_IRQ    
+       KSPIN_LOCK                      lock;
+
+#ifdef SUPPORT_SPINLOCK_ISR    
        PKINTERRUPT     p_int_obj;
-       KIRQL                           irql;
+       KIRQL                           irql;
 #endif
 } spinlock_t;
 
-#ifdef SUPPORT_SPINLOCK_IRQ    
+typedef struct {
+       KLOCK_QUEUE_HANDLE lockh;
+       KIRQL                           irql;
+} spinlockh_t;
+
+#ifdef SUPPORT_SPINLOCK_ISR    
 
 static inline void
-spin_lock_setint( 
+spin_lock_isr_setint( 
        IN              spinlock_t* const       l, 
        IN PKINTERRUPT  p_int_obj )
 {
@@ -20,7 +26,7 @@ spin_lock_setint(
        l->p_int_obj = p_int_obj;
 }
 
-static inline void spin_lock_irq_init(
+static inline void spin_lock_isr_init(
        IN              spinlock_t* const l,
        IN      PKINTERRUPT int_obj
        )
@@ -30,7 +36,7 @@ static inline void spin_lock_irq_init(
 }
 
 static inline unsigned long
-spin_lock_irq( 
+spin_lock_isr( 
        IN              spinlock_t* const       l)
 {
        MT_ASSERT( l );
@@ -39,7 +45,7 @@ spin_lock_irq(
 }
 
 static inline void
-spin_unlock_irq( 
+spin_unlock_isr( 
        IN              spinlock_t* const p_spinlock ) 
 {
        MT_ASSERT( p_spinlock );
@@ -49,7 +55,7 @@ spin_unlock_irq(
 
 #endif
 
-#define SPIN_LOCK_PREP(lh)             KLOCK_QUEUE_HANDLE lh
+#define SPIN_LOCK_PREP(lh)             spinlockh_t lh
 
 static inline void spin_lock_init(
        IN              spinlock_t* const p_spinlock )
@@ -60,20 +66,30 @@ static inline void spin_lock_init(
 static inline void
 spin_lock( 
        IN              spinlock_t* const       l,
-       IN              PKLOCK_QUEUE_HANDLE lockh)
+       IN              spinlockh_t * const lh)
 {
-       MT_ASSERT( l || lockh );
-       ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
-       KeAcquireInStackQueuedSpinLock ( &l->lock, lockh );
+       KIRQL irql = KeGetCurrentIrql();
+
+       MT_ASSERT( l || lh );
+       ASSERT(irql <= DISPATCH_LEVEL);
+
+       if (irql == DISPATCH_LEVEL)
+               KeAcquireInStackQueuedSpinLockAtDpcLevel( &l->lock, &lh->lockh );
+       else
+               KeAcquireInStackQueuedSpinLock( &l->lock, &lh->lockh );
+       lh->irql = irql;
 }
 
 static inline void
 spin_unlock(
-       IN              PKLOCK_QUEUE_HANDLE lockh)
+       IN              spinlockh_t * const lh)
 {
-       MT_ASSERT( lockh );
+       MT_ASSERT( lh );
        ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
-       KeReleaseInStackQueuedSpinLock( lockh );
+       if (lh->irql == DISPATCH_LEVEL)
+               KeReleaseInStackQueuedSpinLockFromDpcLevel( &lh->lockh );
+       else
+               KeReleaseInStackQueuedSpinLock( &lh->lockh );
 }
 
 static inline void
@@ -91,36 +107,36 @@ spin_lock_sync(
 static inline void
 spin_lock_dpc( 
        IN              spinlock_t* const       l,
-       IN              PKLOCK_QUEUE_HANDLE lockh)
+       IN              spinlockh_t * const lh)
 {
-       MT_ASSERT( l || lockh );
+       MT_ASSERT( l || lh );
        ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
-       KeAcquireInStackQueuedSpinLockAtDpcLevel( &l->lock, lockh );
+       KeAcquireInStackQueuedSpinLockAtDpcLevel( &l->lock, &lh->lockh );
 }
 
 /* to be used only at DPC level */
 static inline void
 spin_unlock_dpc(
-       IN              PKLOCK_QUEUE_HANDLE lockh)
+       IN              spinlockh_t * const lh)
 {
        ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
-       KeReleaseInStackQueuedSpinLockFromDpcLevel( lockh );
+       KeReleaseInStackQueuedSpinLockFromDpcLevel( &lh->lockh );
 }
 
 
 /* we are working from DPC level, so we can use usual spinlocks */
-#define spin_lock_irq                                                  spin_lock
+#define spin_lock_irq                                          spin_lock
 #define spin_unlock_irq                                        spin_unlock
 
 /* no diff in Windows */
 #define spin_lock_irqsave                              spin_lock_irq
-#define spin_unlock_irqrestore                 spin_unlock_irq
+#define spin_unlock_irqrestore                         spin_unlock_irq
 
 /* Windows doesn't support such kind of spinlocks so far, but may be tomorrow ... */
-#define rwlock_init                                                            spin_lock_init
+#define rwlock_init                                            spin_lock_init
 #define read_lock_irqsave                              spin_lock_irqsave
-#define read_unlock_irqrestore         spin_unlock_irqrestore
-#define write_lock_irq                                         spin_lock_irq
+#define read_unlock_irqrestore                 spin_unlock_irqrestore
+#define write_lock_irq                                 spin_lock_irq
 #define write_unlock_irq                                       spin_unlock_irq
 
 #endif