Add rmb() and wmb() to <infiniband/arch.h>

Update i386/x86_64 versions to use "lock; addl $0"/"lfence" instead of
just a compiler barrier, to guard against out-of-order speculative
reads.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/ChangeLog b/ChangeLog
index aaf664ec508a118931fc3aebe6bf7a1f8257a40d..971474ab8c9dede204d28abcdb476a3c26cf8684 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2006-10-17  Roland Dreier  <rdreier@cisco.com>
+
+       * include/infiniband/arch.h: Update i386 and x86_64 memory barrier
+       macros to be more than compiler barriers, to guard against
+       out-of-order speculative reads.
+
+       * include/infiniband/arch.h: Add rmb() and wmb() macros in
+       addition to the full mb(), so that low-level drivers can ask for
+       weaker ordering if that's all that is needed.
+       
 2006-10-03  Roland Dreier  <rdreier@cisco.com>
 
        * src/cmd.c (ibv_cmd_get_context_v2, ibv_cmd_get_context)

diff --git a/include/infiniband/arch.h b/include/infiniband/arch.h
index ccb1376d86a5ab4af09cc4dc4907b67d2d845f8e..3aef1b3f8d95896e33227fbba2d819c96402f8fc 100644 (file)
--- a/include/infiniband/arch.h
+++ b/include/infiniband/arch.h
@@ -54,41 +54,65 @@ static inline uint64_t ntohll(uint64_t x) { return x; }
  *
  * mb() - memory barrier.  No loads or stores may be reordered across
  *     this macro by either the compiler or the CPU.
+ * rmb() - read memory barrier.  No loads may be reordered across this
+ *     macro by either the compiler or the CPU.
+ * wmb() - write memory barrier.  No stores may be reordered across
+ *     this macro by either the compiler or the CPU.
  */
 
 #if defined(__i386__)
 
-#define mb()   asm volatile("" ::: "memory")
+#define mb()   asm volatile("lock; addl $0,0(%%esp) " ::: "memory")
+#define rmb()  mb()
+#define wmb()  asm volatile("" ::: "memory")
 
 #elif defined(__x86_64__)
 
-#define mb()   asm volatile("" ::: "memory")
+/*
+ * Only use lfence for mb() and rmb() because we don't care about
+ * ordering against non-temporal stores (for now at least).
+ */
+#define mb()   asm volatile("lfence" ::: "memory")
+#define rmb()  mb()
+#define wmb()  asm volatile("" ::: "memory")
 
 #elif defined(__PPC64__)
 
 #define mb()   asm volatile("sync" ::: "memory")
+#define rmb()  asm volatile("lwsync" ::: "memory")
+#define wmb()  mb()
 
 #elif defined(__ia64__)
 
 #define mb()   asm volatile("mf" ::: "memory")
+#define rmb()  mb()
+#define wmb()  mb()
 
 #elif defined(__PPC__)
 
 #define mb()   asm volatile("sync" ::: "memory")
+#define rmb()  mb()
+#define wmb()  asm volatile("eieio" ::: "memory")
 
 #elif defined(__sparc_v9__)
 
 #define mb()   asm volatile("membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad" ::: "memory")
+#define rmb()  asm volatile("membar #LoadLoad" ::: "memory")
+#define wmb()  asm volatile("membar #StoreStore" ::: "memory")
 
 #elif defined(__sparc__)
 
 #define mb()   asm volatile("" ::: "memory")
+#define rmb()  mb()
+#define wmb()  mb()
 
 #else
 
 #warning No architecture specific defines found.  Using generic implementation.
 
 #define mb()   asm volatile("" ::: "memory")
+#define rmb()  mb()
+#define wmb()  mb()
 
 #endif