From: sleybo <sleybo@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Date: Wed, 7 Feb 2007 12:58:38 +0000 (+0000)
Subject: [MTHCA] support for in-line send in the kernel
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=9d1496a9e4c0869ca89a447c06643112f858cd67;p=~shefty%2Frdma-win.git

[MTHCA] support for in-line send in the kernel

git-svn-id: svn://openib.tc.cornell.edu/gen1@582 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86
---

diff --git a/trunk/hw/mthca/kernel/mthca_qp.c b/trunk/hw/mthca/kernel/mthca_qp.c
index 330d37e7..8232bcc3 100644
--- a/trunk/hw/mthca/kernel/mthca_qp.c
+++ b/trunk/hw/mthca/kernel/mthca_qp.c
@@ -339,7 +339,8 @@ static void dump_wqe(u32 print_lvl, u32 *wqe_ptr , struct mthca_qp *qp_ptr)
 {
 	__be32 *wqe = wqe_ptr;
 
-	UNREFERENCED_PARAMETER(qp_ptr);
+	UNUSED_PARAM_WOWPP(qp_ptr);
+	UNUSED_PARAM_WOWPP(print_lvl);
 
 	(void) wqe;	/* avoid warning if mthca_dbg compiled away... */
 	HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents  QPN 0x%06x \n",qp_ptr->qpn));
@@ -906,20 +907,18 @@ static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int d
 		return max_data_size;
 }
 
-static inline int mthca_max_inline_data(struct mthca_pd *pd, int max_data_size)
+static inline int mthca_max_inline_data(int max_data_size)
 {
-	/* We don't support inline data for kernel QPs (yet). */
-	return pd->ibpd.ucontext ? max_data_size - MTHCA_INLINE_HEADER_SIZE : 0;
+	return max_data_size - MTHCA_INLINE_HEADER_SIZE ;
 }
 
 static void mthca_adjust_qp_caps(struct mthca_dev *dev,
-				 struct mthca_pd *pd,
 				 struct mthca_qp *qp)
 {
 	int max_data_size = mthca_max_data_size(dev, qp,
 		min(dev->limits.max_desc_sz, 1 << qp->sq.wqe_shift));
 
-	qp->max_inline_data = mthca_max_inline_data(pd, max_data_size);
+	qp->max_inline_data = mthca_max_inline_data( max_data_size);
 
 	qp->sq.max_gs = min(dev->limits.max_sg,
 		(int)(max_data_size / sizeof (struct mthca_data_seg)));
@@ -1142,7 +1141,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
 		return ret;
 	}
 
-	mthca_adjust_qp_caps(dev, pd, qp);
+	mthca_adjust_qp_caps(dev, qp);
 
 	/*
 	 * If this is a userspace QP, we're done now.  The doorbells
@@ -1192,7 +1191,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
 }
 
 static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,
-					 struct mthca_pd *pd, struct mthca_qp *qp)
+	struct mthca_qp *qp)
 {
 	int max_data_size = mthca_max_data_size(dev, qp, dev->limits.max_desc_sz);
 
@@ -1201,7 +1200,7 @@ static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,
 	    cap->max_recv_wr  	 > (u32)dev->limits.max_wqes ||
 	    cap->max_send_sge 	 > (u32)dev->limits.max_sg   ||
 	    cap->max_recv_sge 	 > (u32)dev->limits.max_sg   ||
-	    cap->max_inline_data > (u32)mthca_max_inline_data(pd, max_data_size))
+	    cap->max_inline_data > (u32)mthca_max_inline_data(max_data_size))
 		return -EINVAL;
 
 	/*
@@ -1249,7 +1248,7 @@ int mthca_alloc_qp(struct mthca_dev *dev,
 	default: return -EINVAL;
 	}
 
-	err = mthca_set_qp_size(dev, cap, pd, qp);
+	err = mthca_set_qp_size(dev, cap, qp);
 	if (err)
 		return err;
 
@@ -1288,7 +1287,7 @@ int mthca_alloc_sqp(struct mthca_dev *dev,
 	SPIN_LOCK_PREP(lhr);
 	SPIN_LOCK_PREP(lht);
 
-	err = mthca_set_qp_size(dev, cap, pd, &sqp->qp);
+	err = mthca_set_qp_size(dev, cap, &sqp->qp);
 	if (err)
 		return err;
 
@@ -1518,9 +1517,9 @@ static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq,
 		return 0;
 
 	cq = to_mcq(ib_cq);
-	spin_lock(&cq->lock, &lh);
+	spin_lock_dpc(&cq->lock, &lh);
 	cur = wq->head - wq->tail;
-	spin_unlock(&lh);
+	spin_unlock_dpc(&lh);
 
 	return (int)cur + nreq >= wq->max;
 }
@@ -1541,10 +1540,10 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
 	int ind;
 	u8 op0 = 0;
 	enum ib_wr_opcode opcode;
-	SPIN_LOCK_PREP(lh);
+	SPIN_LOCK_PREP(lh);   
 
 	spin_lock_irqsave(&qp->sq.lock, &lh);
-
+    
 	/* XXX check that state is OK to post send */
 
 	ind = qp->sq.next_ind;
@@ -1682,19 +1681,47 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
 				*bad_wr = wr;
 			goto out;
 		}
+		if (wr->send_opt & IB_SEND_OPT_INLINE) {
+			if (wr->num_ds) {
+				struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;
+				uint32_t s = 0;
+
+				wqe += sizeof *seg;
+				for (i = 0; i < (int)wr->num_ds; ++i) {
+					struct _ib_local_ds *sge = &wr->ds_array[i];
+
+					s += sge->length;
+
+					if (s > (uint32_t)qp->max_inline_data) {
+						err = -EINVAL;
+						if (bad_wr)
+							*bad_wr = wr;
+						goto out;
+					}
+
+					memcpy(wqe, (void *) (ULONG_PTR) sge->vaddr,
+					       sge->length);
+					wqe += sge->length;
+				}
 
-		for (i = 0; i < (int)wr->num_ds; ++i) {
-			((struct mthca_data_seg *) wqe)->byte_count =
-				cl_hton32(wr->ds_array[i].length);
-			((struct mthca_data_seg *) wqe)->lkey =
-				cl_hton32(wr->ds_array[i].lkey);
-			((struct mthca_data_seg *) wqe)->addr =
-				cl_hton64(wr->ds_array[i].vaddr);
-			wqe += sizeof (struct mthca_data_seg);
-			size += sizeof (struct mthca_data_seg) / 16;
-			HCA_PRINT(TRACE_LEVEL_VERBOSE ,HCA_DBG_QP ,("SQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x%x\n",qp->qpn,i,
-				(wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length));
-		}
+				seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);
+				size += align(s + sizeof *seg, 16) / 16;
+			}
+		} else {
+    			
+    		for (i = 0; i < (int)wr->num_ds; ++i) {
+    			((struct mthca_data_seg *) wqe)->byte_count =
+    				cl_hton32(wr->ds_array[i].length);
+    			((struct mthca_data_seg *) wqe)->lkey =
+    				cl_hton32(wr->ds_array[i].lkey);
+    			((struct mthca_data_seg *) wqe)->addr =
+    				cl_hton64(wr->ds_array[i].vaddr);
+    			wqe += sizeof (struct mthca_data_seg);
+    			size += sizeof (struct mthca_data_seg) / 16;
+    			HCA_PRINT(TRACE_LEVEL_VERBOSE ,HCA_DBG_QP ,("SQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x%x\n",qp->qpn,i,
+    				(wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length));
+    		}
+    	}
 
 		/* Add one more inline data segment for ICRC */
 		if (qp->transport == MLX) {
@@ -1753,8 +1780,8 @@ out:
 
 	qp->sq.next_ind = ind;
 	qp->sq.head    += nreq;
-
-	spin_unlock_irqrestore(&lh);
+	
+    spin_unlock_irqrestore(&lh);   
 	return err;
 }
 
@@ -2054,17 +2081,44 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
 				*bad_wr = wr;
 			goto out;
 		}
+        if (wr->send_opt & IB_SEND_OPT_INLINE) {
+			if (wr->num_ds) {
+				struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;
+				uint32_t s = 0;
+
+				wqe += sizeof *seg;
+				for (i = 0; i < (int)wr->num_ds; ++i) {
+					struct _ib_local_ds *sge = &wr->ds_array[i];
+
+					s += sge->length;
+
+					if (s > (uint32_t)qp->max_inline_data) {
+						err = -EINVAL;
+						if (bad_wr)
+							*bad_wr = wr;
+						goto out;
+					}
+
+					memcpy(wqe, (void *) (uintptr_t) sge->vaddr,
+					       sge->length);
+					wqe += sge->length;
+				}
 
-		for (i = 0; i < (int)wr->num_ds; ++i) {
-			((struct mthca_data_seg *) wqe)->byte_count =
-				cl_hton32(wr->ds_array[i].length);
-			((struct mthca_data_seg *) wqe)->lkey =
-				cl_hton32(wr->ds_array[i].lkey);
-			((struct mthca_data_seg *) wqe)->addr =
-				cl_hton64(wr->ds_array[i].vaddr);
-			wqe += sizeof (struct mthca_data_seg);
-			size += sizeof (struct mthca_data_seg) / 16;
-		}
+				seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);
+				size += align(s + sizeof *seg, 16) / 16;
+			}
+		} else {
+    		for (i = 0; i < (int)wr->num_ds; ++i) {
+    			((struct mthca_data_seg *) wqe)->byte_count =
+    				cl_hton32(wr->ds_array[i].length);
+    			((struct mthca_data_seg *) wqe)->lkey =
+    				cl_hton32(wr->ds_array[i].lkey);
+    			((struct mthca_data_seg *) wqe)->addr =
+    				cl_hton64(wr->ds_array[i].vaddr);
+    			wqe += sizeof (struct mthca_data_seg);
+    			size += sizeof (struct mthca_data_seg) / 16;
+    		}
+    	}
 
 		/* Add one more inline data segment for ICRC */
 		if (qp->transport == MLX) {
diff --git a/trunk/hw/mthca/mthca_wqe.h b/trunk/hw/mthca/mthca_wqe.h
index fd3e5b91..ff3aab48 100644
--- a/trunk/hw/mthca/mthca_wqe.h
+++ b/trunk/hw/mthca/mthca_wqe.h
@@ -127,5 +127,11 @@ struct mthca_inline_seg {
 	uint32_t	byte_count;
 };
 
+
+static inline unsigned long align(unsigned long val, unsigned long align)
+{
+	return (val + align - 1) & ~(align - 1);
+}
+
 #endif /* MTHCA_WQE_H */
 
diff --git a/trunk/hw/mthca/user/mlnx_uvp.h b/trunk/hw/mthca/user/mlnx_uvp.h
index 6470a398..f0c1d438 100644
--- a/trunk/hw/mthca/user/mlnx_uvp.h
+++ b/trunk/hw/mthca/user/mlnx_uvp.h
@@ -212,10 +212,6 @@ struct mthca_ah_page {
 	unsigned      	      free[0];
 };
 
-static inline unsigned long align(unsigned long val, unsigned long align)
-{
-	return (val + align - 1) & ~(align - 1);
-}
 
 static inline uintptr_t db_align(uint32_t *db)
 {
diff --git a/trunk/hw/mthca/user/mlnx_uvp_verbs.c b/trunk/hw/mthca/user/mlnx_uvp_verbs.c
index 528222fd..445b2838 100644
--- a/trunk/hw/mthca/user/mlnx_uvp_verbs.c
+++ b/trunk/hw/mthca/user/mlnx_uvp_verbs.c
@@ -37,6 +37,8 @@
 
 #include "mlnx_uvp.h"
 #include "mx_abi.h"
+#include "mthca_wqe.h"
+
 
 #if defined(EVENT_TRACING)
 #include "mlnx_uvp_verbs.tmh"