Add support to join IPOIB multicast groups

Add to the librdmacm an IPOIB port space that allows interoperability with
IPOIB multicast traffic.  Use of the RDMA_PS_IPOIB is limited to multicast
join/leave.

Rename the RDMA_UD_QKEY to RDMA_UDP_QKEY to signify that the qkey is only
used with the RDMA_PS_IPOIB port space.  Update mckey test program based on
patch from Or Gerlitz.

Signed-off-by: Or Gerlitz <ogerlitz@voltaire.com>
Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/examples/mckey.c b/examples/mckey.c
index 39f77d7d6f9b2feff257b1c7d4c0a924233e136c..d4f83b8f4493a423f25245d8dc2f7a6feb25ebc6 100644 (file)
--- a/examples/mckey.c
+++ b/examples/mckey.c
@@ -78,6 +78,7 @@ static int message_count = 10;
 static int is_sender;
 static char *dst_addr;
 static char *src_addr;
+static enum rdma_port_space port_space = RDMA_PS_UDP;
 
 static int create_message(struct cmatest_node *node)
 {
@@ -328,7 +329,7 @@ static int alloc_nodes(void)
        for (i = 0; i < connections; i++) {
                test.nodes[i].id = i;
                ret = rdma_create_id(test.channel, &test.nodes[i].cma_id,
-                                    &test.nodes[i], RDMA_PS_UDP);
+                                    &test.nodes[i], port_space);
                if (ret)
                        goto err;
        }
@@ -478,7 +479,7 @@ int main(int argc, char **argv)
 {
        int op, ret;
 
-       while ((op = getopt(argc, argv, "m:sb:c:C:S:")) != -1) {
+       while ((op = getopt(argc, argv, "m:sb:c:C:S:p:")) != -1) {
                switch (op) {
                case 'm':
                        dst_addr = optarg;
@@ -498,6 +499,9 @@ int main(int argc, char **argv)
                case 'S':
                        message_size = atoi(optarg);
                        break;
+               case 'p':
+                       port_space = strtol(optarg, NULL, 0);
+                       break;
                default:
                        printf("usage: %s\n", argv[0]);
                        printf("\t-m multicast_address\n");
@@ -506,6 +510,8 @@ int main(int argc, char **argv)
                        printf("\t[-c connections]\n");
                        printf("\t[-C message_count]\n");
                        printf("\t[-S message_size]\n");
+                       printf("\t[-p port_space - %#x for UDP (default), "
+                              "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);
                        exit(1);
                }
        }

diff --git a/examples/udaddy.c b/examples/udaddy.c
index ab9ace69154419176cd3846cc00dcf040d52c93f..153e39c7dac13d761a5fcaefb1ab2681eac1defc 100644 (file)
--- a/examples/udaddy.c
+++ b/examples/udaddy.c
@@ -420,7 +420,7 @@ static void create_reply_ah(struct cmatest_node *node, struct ibv_wc *wc)
        node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem,
                                         node->cma_id->port_num);
        node->remote_qpn = ntohl(wc->imm_data);
-       node->remote_qkey = RDMA_UD_QKEY;
+       node->remote_qkey = RDMA_UDP_QKEY;
 }
 
 static int poll_cqs(void)

diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 88a25b2e07d157746acd7a954220e3d8c72d6f54..e30f8cd21df0476bb9ec6389829c54ebd900ed50 100644 (file)
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -61,15 +61,16 @@ enum rdma_cm_event_type {
 };
 
 enum rdma_port_space {
+       RDMA_PS_IPOIB= 0x0002,
        RDMA_PS_TCP  = 0x0106,
        RDMA_PS_UDP  = 0x0111,
 };
 
 /*
- * Global qkey value for all UD QPs and multicast groups created via the 
+ * Global qkey value for UDP QPs and multicast groups created via the 
  * RDMA CM.
  */
-#define RDMA_UD_QKEY 0x01234567
+#define RDMA_UDP_QKEY 0x01234567
 
 struct ib_addr {
        union ibv_gid   sgid;

diff --git a/src/cma.c b/src/cma.c
index 7ab685bf434a95f686993a2bbe812af48896411f..6a0d076f262cb0434483772d21f33e5ab47ce120 100644 (file)
--- a/src/cma.c
+++ b/src/cma.c
@@ -698,7 +698,7 @@ static int ucma_init_ud_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
 
        qp_attr.port_num = id_priv->id.port_num;
        qp_attr.qp_state = IBV_QPS_INIT;
-       qp_attr.qkey = RDMA_UD_QKEY;
+       qp_attr.qkey = RDMA_UDP_QKEY;   /* Will override PS_IPOIB on join */
        ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
                                          IBV_QP_PORT | IBV_QP_QKEY);
        if (ret)
@@ -729,7 +729,7 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
        if (!qp)
                return -ENOMEM;
 
-       if (id->ps == RDMA_PS_UDP)
+       if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
                ret = ucma_init_ud_qp(id_priv, qp);
        else
                ret = ucma_init_ib_qp(id_priv, qp);
@@ -1136,14 +1136,25 @@ static int ucma_process_establish(struct rdma_cm_id *id)
 
 static int ucma_process_join(struct cma_event *evt)
 {
+       int ret;
+
        evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
        evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;
 
-       if (evt->id_priv->id.qp)
-               return ibv_attach_mcast(evt->id_priv->id.qp,
-                                       &evt->mc->mgid, evt->mc->mlid);
-       else
+       if (!evt->id_priv->id.qp)
                return 0;
+
+       if (evt->id_priv->id.ps == RDMA_PS_IPOIB) {
+               struct ibv_qp_attr qp_attr;
+
+               qp_attr.qkey = evt->event.param.ud.qkey;
+               ret = ibv_modify_qp(evt->id_priv->id.qp, &qp_attr, IBV_QP_QKEY);
+               if (ret)
+                       return ret;
+       }
+
+       return ibv_attach_mcast(evt->id_priv->id.qp, &evt->mc->mgid,
+                               evt->mc->mlid);
 }
 
 static void ucma_copy_conn_event(struct cma_event *event,