[MTHCA] support for in-line send in the kernel

git-svn-id: svn://openib.tc.cornell.edu/gen1@582 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/hw/mthca/kernel/mthca_qp.c b/trunk/hw/mthca/kernel/mthca_qp.c
index 330d37e7833e9482d1d14af3aa4a82bda101beea..8232bcc357e7eb6a8ab8e428238d8f40e24a5c98 100644 (file)
--- a/trunk/hw/mthca/kernel/mthca_qp.c
+++ b/trunk/hw/mthca/kernel/mthca_qp.c
@@ -339,7 +339,8 @@ static void dump_wqe(u32 print_lvl, u32 *wqe_ptr , struct mthca_qp *qp_ptr)
 {
        __be32 *wqe = wqe_ptr;
 
-       UNREFERENCED_PARAMETER(qp_ptr);
+       UNUSED_PARAM_WOWPP(qp_ptr);
+       UNUSED_PARAM_WOWPP(print_lvl);
 
        (void) wqe;     /* avoid warning if mthca_dbg compiled away... */
        HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents  QPN 0x%06x \n",qp_ptr->qpn));
@@ -906,20 +907,18 @@ static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int d
                return max_data_size;
 }
 
-static inline int mthca_max_inline_data(struct mthca_pd *pd, int max_data_size)
+static inline int mthca_max_inline_data(int max_data_size)
 {
-       /* We don't support inline data for kernel QPs (yet). */
-       return pd->ibpd.ucontext ? max_data_size - MTHCA_INLINE_HEADER_SIZE : 0;
+       return max_data_size - MTHCA_INLINE_HEADER_SIZE ;
 }
 
 static void mthca_adjust_qp_caps(struct mthca_dev *dev,
-                                struct mthca_pd *pd,
                                 struct mthca_qp *qp)
 {
        int max_data_size = mthca_max_data_size(dev, qp,
                min(dev->limits.max_desc_sz, 1 << qp->sq.wqe_shift));
 
-       qp->max_inline_data = mthca_max_inline_data(pd, max_data_size);
+       qp->max_inline_data = mthca_max_inline_data( max_data_size);
 
        qp->sq.max_gs = min(dev->limits.max_sg,
                (int)(max_data_size / sizeof (struct mthca_data_seg)));
@@ -1142,7 +1141,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
                return ret;
        }
 
-       mthca_adjust_qp_caps(dev, pd, qp);
+       mthca_adjust_qp_caps(dev, qp);
 
        /*
         * If this is a userspace QP, we're done now.  The doorbells
@@ -1192,7 +1191,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
 }
 
 static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,
-                                        struct mthca_pd *pd, struct mthca_qp *qp)
+       struct mthca_qp *qp)
 {
        int max_data_size = mthca_max_data_size(dev, qp, dev->limits.max_desc_sz);
 
@@ -1201,7 +1200,7 @@ static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,
            cap->max_recv_wr     > (u32)dev->limits.max_wqes ||
            cap->max_send_sge    > (u32)dev->limits.max_sg   ||
            cap->max_recv_sge    > (u32)dev->limits.max_sg   ||
-           cap->max_inline_data > (u32)mthca_max_inline_data(pd, max_data_size))
+           cap->max_inline_data > (u32)mthca_max_inline_data(max_data_size))
                return -EINVAL;
 
        /*
@@ -1249,7 +1248,7 @@ int mthca_alloc_qp(struct mthca_dev *dev,
        default: return -EINVAL;
        }
 
-       err = mthca_set_qp_size(dev, cap, pd, qp);
+       err = mthca_set_qp_size(dev, cap, qp);
        if (err)
                return err;
 
@@ -1288,7 +1287,7 @@ int mthca_alloc_sqp(struct mthca_dev *dev,
        SPIN_LOCK_PREP(lhr);
        SPIN_LOCK_PREP(lht);
 
-       err = mthca_set_qp_size(dev, cap, pd, &sqp->qp);
+       err = mthca_set_qp_size(dev, cap, &sqp->qp);
        if (err)
                return err;
 
@@ -1518,9 +1517,9 @@ static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq,
                return 0;
 
        cq = to_mcq(ib_cq);
-       spin_lock(&cq->lock, &lh);
+       spin_lock_dpc(&cq->lock, &lh);
        cur = wq->head - wq->tail;
-       spin_unlock(&lh);
+       spin_unlock_dpc(&lh);
 
        return (int)cur + nreq >= wq->max;
 }
@@ -1541,10 +1540,10 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
        int ind;
        u8 op0 = 0;
        enum ib_wr_opcode opcode;
-       SPIN_LOCK_PREP(lh);
+       SPIN_LOCK_PREP(lh);   
 
        spin_lock_irqsave(&qp->sq.lock, &lh);
-
+    
        /* XXX check that state is OK to post send */
 
        ind = qp->sq.next_ind;
@@ -1682,19 +1681,47 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
                                *bad_wr = wr;
                        goto out;
                }
+               if (wr->send_opt & IB_SEND_OPT_INLINE) {
+                       if (wr->num_ds) {
+                               struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;
+                               uint32_t s = 0;
+
+                               wqe += sizeof *seg;
+                               for (i = 0; i < (int)wr->num_ds; ++i) {
+                                       struct _ib_local_ds *sge = &wr->ds_array[i];
+
+                                       s += sge->length;
+
+                                       if (s > (uint32_t)qp->max_inline_data) {
+                                               err = -EINVAL;
+                                               if (bad_wr)
+                                                       *bad_wr = wr;
+                                               goto out;
+                                       }
+
+                                       memcpy(wqe, (void *) (ULONG_PTR) sge->vaddr,
+                                              sge->length);
+                                       wqe += sge->length;
+                               }
 
-               for (i = 0; i < (int)wr->num_ds; ++i) {
-                       ((struct mthca_data_seg *) wqe)->byte_count =
-                               cl_hton32(wr->ds_array[i].length);
-                       ((struct mthca_data_seg *) wqe)->lkey =
-                               cl_hton32(wr->ds_array[i].lkey);
-                       ((struct mthca_data_seg *) wqe)->addr =
-                               cl_hton64(wr->ds_array[i].vaddr);
-                       wqe += sizeof (struct mthca_data_seg);
-                       size += sizeof (struct mthca_data_seg) / 16;
-                       HCA_PRINT(TRACE_LEVEL_VERBOSE ,HCA_DBG_QP ,("SQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x%x\n",qp->qpn,i,
-                               (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length));
-               }
+                               seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);
+                               size += align(s + sizeof *seg, 16) / 16;
+                       }
+               } else {
+                       
+               for (i = 0; i < (int)wr->num_ds; ++i) {
+                       ((struct mthca_data_seg *) wqe)->byte_count =
+                               cl_hton32(wr->ds_array[i].length);
+                       ((struct mthca_data_seg *) wqe)->lkey =
+                               cl_hton32(wr->ds_array[i].lkey);
+                       ((struct mthca_data_seg *) wqe)->addr =
+                               cl_hton64(wr->ds_array[i].vaddr);
+                       wqe += sizeof (struct mthca_data_seg);
+                       size += sizeof (struct mthca_data_seg) / 16;
+                       HCA_PRINT(TRACE_LEVEL_VERBOSE ,HCA_DBG_QP ,("SQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x%x\n",qp->qpn,i,
+                               (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length));
+               }
+       }
 
                /* Add one more inline data segment for ICRC */
                if (qp->transport == MLX) {
@@ -1753,8 +1780,8 @@ out:
 
        qp->sq.next_ind = ind;
        qp->sq.head    += nreq;
-
-       spin_unlock_irqrestore(&lh);
+       
+    spin_unlock_irqrestore(&lh);   
        return err;
 }
 
@@ -2054,17 +2081,44 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
                                *bad_wr = wr;
                        goto out;
                }
+        if (wr->send_opt & IB_SEND_OPT_INLINE) {
+                       if (wr->num_ds) {
+                               struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;
+                               uint32_t s = 0;
+
+                               wqe += sizeof *seg;
+                               for (i = 0; i < (int)wr->num_ds; ++i) {
+                                       struct _ib_local_ds *sge = &wr->ds_array[i];
+
+                                       s += sge->length;
+
+                                       if (s > (uint32_t)qp->max_inline_data) {
+                                               err = -EINVAL;
+                                               if (bad_wr)
+                                                       *bad_wr = wr;
+                                               goto out;
+                                       }
+
+                                       memcpy(wqe, (void *) (uintptr_t) sge->vaddr,
+                                              sge->length);
+                                       wqe += sge->length;
+                               }
 
-               for (i = 0; i < (int)wr->num_ds; ++i) {
-                       ((struct mthca_data_seg *) wqe)->byte_count =
-                               cl_hton32(wr->ds_array[i].length);
-                       ((struct mthca_data_seg *) wqe)->lkey =
-                               cl_hton32(wr->ds_array[i].lkey);
-                       ((struct mthca_data_seg *) wqe)->addr =
-                               cl_hton64(wr->ds_array[i].vaddr);
-                       wqe += sizeof (struct mthca_data_seg);
-                       size += sizeof (struct mthca_data_seg) / 16;
-               }
+                               seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);
+                               size += align(s + sizeof *seg, 16) / 16;
+                       }
+               } else {
+               for (i = 0; i < (int)wr->num_ds; ++i) {
+                       ((struct mthca_data_seg *) wqe)->byte_count =
+                               cl_hton32(wr->ds_array[i].length);
+                       ((struct mthca_data_seg *) wqe)->lkey =
+                               cl_hton32(wr->ds_array[i].lkey);
+                       ((struct mthca_data_seg *) wqe)->addr =
+                               cl_hton64(wr->ds_array[i].vaddr);
+                       wqe += sizeof (struct mthca_data_seg);
+                       size += sizeof (struct mthca_data_seg) / 16;
+               }
+       }
 
                /* Add one more inline data segment for ICRC */
                if (qp->transport == MLX) {

diff --git a/trunk/hw/mthca/mthca_wqe.h b/trunk/hw/mthca/mthca_wqe.h
index fd3e5b910626cda02ca373e2bade9244543438ba..ff3aab48a78c74764a7221202d0c59180484ee26 100644 (file)
--- a/trunk/hw/mthca/mthca_wqe.h
+++ b/trunk/hw/mthca/mthca_wqe.h
@@ -127,5 +127,11 @@ struct mthca_inline_seg {
        uint32_t        byte_count;\r
 };\r
 \r
+\r
+static inline unsigned long align(unsigned long val, unsigned long align)\r
+{\r
+       return (val + align - 1) & ~(align - 1);\r
+}\r
+\r
 #endif /* MTHCA_WQE_H */\r
 \r

diff --git a/trunk/hw/mthca/user/mlnx_uvp.h b/trunk/hw/mthca/user/mlnx_uvp.h
index 6470a398c647f0b6674f16ff74035b6b149b94fb..f0c1d438f2e53246d3ff45f4376d91b199a5043e 100644 (file)
--- a/trunk/hw/mthca/user/mlnx_uvp.h
+++ b/trunk/hw/mthca/user/mlnx_uvp.h
@@ -212,10 +212,6 @@ struct mthca_ah_page {
        unsigned              free[0];
 };
 
-static inline unsigned long align(unsigned long val, unsigned long align)
-{
-       return (val + align - 1) & ~(align - 1);
-}
 
 static inline uintptr_t db_align(uint32_t *db)
 {

diff --git a/trunk/hw/mthca/user/mlnx_uvp_verbs.c b/trunk/hw/mthca/user/mlnx_uvp_verbs.c
index 528222fd9084d00d987c300e4c01a6ebb09013a5..445b283890730c348fc3f8b07578dcc3b228a547 100644 (file)
--- a/trunk/hw/mthca/user/mlnx_uvp_verbs.c
+++ b/trunk/hw/mthca/user/mlnx_uvp_verbs.c
@@ -37,6 +37,8 @@
 
 #include "mlnx_uvp.h"
 #include "mx_abi.h"
+#include "mthca_wqe.h"
+
 
 #if defined(EVENT_TRACING)
 #include "mlnx_uvp_verbs.tmh"