cma: Workaround for rdma_ucm kernel bug

For certain new kernels, IB_QP_SMAC bit is erroneously
set in QP attribute mask. This workaround turns
off that bit. It allows SSA connections (AF_IB)
to work. Without this workaround, there are issues
on the client side.

Kernel patch which caused issue is commit dd5f03b
IB/core: Ethernet L2 attributes in verbs/cm structures

Kernel patch to fix this in upstream:

commit c2be9dc0e0fa59cc43c2c7084fc42b430809a0fe
Author: Ilya Nelkenbaum <ilyan@mellanox.com>
Date:   Thu Feb 5 13:53:48 2015 +0200

    IB/core: When marshaling ucma path from user-space, clear unused fields

    When marshaling a user path to the kernel struct ib_sa_path, we need
    to zero smac and dmac and set the vlan id to the "no vlan" value.

    This is to ensure that Ethernet attributes are not used with
    InfiniBand QPs.

    Fixes: dd5f03beb4f7 ("IB/core: Ethernet L2 attributes in verbs/cm structures")
    Signed-off-by: Ilya Nelkenbaum <ilyan@mellanox.com>
    Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
    Signed-off-by: Roland Dreier <roland@purestorage.com>
The fix was pushed to stable 3.14, 3.18 and 3.19 versions.

Signed-off-by: Ilya Nelkenbaum <ilyan@mellanox.com>
Signed-off-by: Hal Rosenstock <hal@mellanox.com>
Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/src/cma.c b/src/cma.c
index 749140ecbfd98db074ff303b501bb5dec2a51f21..bdbd7e8b6ce9b52e866f613cd30018444e68e3aa 100644 (file)
--- a/src/cma.c
+++ b/src/cma.c
@@ -49,6 +49,7 @@
 #include <stddef.h>
 #include <netdb.h>
 #include <syslog.h>
+#include <limits.h>
 
 #include "cma.h"
 #include "indexer.h"
@@ -73,10 +74,15 @@ do {                                                \
        (req)->response = (uintptr_t) (resp);   \
 } while (0)
 
+struct cma_port {
+       uint8_t                 link_layer;
+};
+
 struct cma_device {
        struct ibv_context *verbs;
        struct ibv_pd      *pd;
        struct ibv_xrcd    *xrcd;
+       struct cma_port    *port;
        uint64_t            guid;
        int                 port_cnt;
        int                 refcnt;
@@ -142,6 +148,7 @@ static void ucma_cleanup(void)
                        if (cma_dev_array[cma_dev_cnt].refcnt)
                                ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
                        ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
+                       free(cma_dev_array[cma_dev_cnt].port);
                        cma_init_cnt--;
                }
 
@@ -292,8 +299,9 @@ static struct ibv_context *ucma_open_device(uint64_t guid)
 
 static int ucma_init_device(struct cma_device *cma_dev)
 {
+       struct ibv_port_attr port_attr;
        struct ibv_device_attr attr;
-       int ret;
+       int i, ret;
 
        if (cma_dev->verbs)
                return 0;
@@ -309,6 +317,19 @@ static int ucma_init_device(struct cma_device *cma_dev)
                goto err;
        }
 
+       cma_dev->port = malloc(sizeof *cma_dev->port * attr.phys_port_cnt);
+       if (!cma_dev->port) {
+               ret = ERR(ENOMEM);
+               goto err;
+       }
+
+       for (i = 1; i <= attr.phys_port_cnt; i++) {
+               if (ibv_query_port(cma_dev->verbs, i, &port_attr))
+                       cma_dev->port[i - 1].link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+               else
+                       cma_dev->port[i - 1].link_layer = port_attr.link_layer;
+       }
+
        cma_dev->port_cnt = attr.phys_port_cnt;
        cma_dev->max_qpsize = attr.max_qp_wr;
        cma_dev->max_initiator_depth = (uint8_t) attr.max_qp_init_rd_atom;
@@ -1034,8 +1055,10 @@ static int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr,
 
 static int ucma_modify_qp_rtr(struct rdma_cm_id *id, uint8_t resp_res)
 {
+       struct cma_id_private *id_priv;
        struct ibv_qp_attr qp_attr;
        int qp_attr_mask, ret;
+       uint8_t link_layer;
 
        if (!id->qp)
                return ERR(EINVAL);
@@ -1055,6 +1078,16 @@ static int ucma_modify_qp_rtr(struct rdma_cm_id *id, uint8_t resp_res)
        if (ret)
                return ret;
 
+       /*
+        * Workaround for rdma_ucm kernel bug:
+        * mask off qp_attr_mask bits 21-24 which are used for RoCE
+        */
+       id_priv = container_of(id, struct cma_id_private, id);
+       link_layer = id_priv->cma_dev->port[id->port_num - 1].link_layer;
+
+       if (link_layer == IBV_LINK_LAYER_INFINIBAND)
+               qp_attr_mask &= UINT_MAX ^ 0xe00000;
+
        if (resp_res != RDMA_MAX_RESP_RES)
                qp_attr.max_dest_rd_atomic = resp_res;
        return rdma_seterrno(ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask));