Add multicast support to librdmacm

Add support for multicast QPs through the rdma cm library.

diff --git a/Makefile.am b/Makefile.am
index 6a1f5795a144119094ff53b204f806485c968de9..d4caf796de29c5f17d48ab56e7073a07967362fb 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,16 +18,15 @@ endif
 src_librdmacm_la_SOURCES = src/cma.c
 src_librdmacm_la_LDFLAGS = -avoid-version $(rdmacm_version_script)
 
-bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy
-#examples/mckey
+bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey
 examples_ucmatose_SOURCES = examples/cmatose.c
 examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
 examples_rping_SOURCES = examples/rping.c
 examples_rping_LDADD = $(top_builddir)/src/librdmacm.la
 examples_udaddy_SOURCES = examples/udaddy.c
 examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la
-#examples_mckey_SOURCES = examples/mckey.c
-#examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
+examples_mckey_SOURCES = examples/mckey.c
+examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
 
 librdmacmincludedir = $(includedir)/rdma
 

diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 711348ac1d11e8f08ddead325fb6cb819208d1c4..88a25b2e07d157746acd7a954220e3d8c72d6f54 100644 (file)
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -55,7 +55,9 @@ enum rdma_cm_event_type {
        RDMA_CM_EVENT_REJECTED,
        RDMA_CM_EVENT_ESTABLISHED,
        RDMA_CM_EVENT_DISCONNECTED,
-       RDMA_CM_EVENT_DEVICE_REMOVAL
+       RDMA_CM_EVENT_DEVICE_REMOVAL,
+       RDMA_CM_EVENT_MULTICAST_JOIN,
+       RDMA_CM_EVENT_MULTICAST_ERROR
 };
 
 enum rdma_port_space {
@@ -279,6 +281,24 @@ int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event);
  */
 int rdma_disconnect(struct rdma_cm_id *id);
 
+/**
+ * rdma_join_multicast - Join the multicast group specified by the given
+ *   address.
+ * @id: Communication identifier associated with the request.
+ * @addr: Multicast address identifying the group to join.
+ * @context: User-defined context associated with the join request.  The
+ *   context is returned to the user through the private_data field in
+ *   the rdma_cm_event.
+ */
+int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
+                       void *context);
+
+/**
+ * rdma_leave_multicast - Leave the multicast group specified by the given
+ *   address.
+ */
+int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr);
+
 /**
  * rdma_get_cm_event - Retrieves the next pending communications event,
  *   if no event is pending waits for an event.

diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index cd2d0aad3735652d0b516982d3aa59ae3a4bc258..2ffcf929ce8cd59245db25c6df458ca408c004e2 100644 (file)
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -41,7 +41,7 @@
  */
 
 #define RDMA_USER_CM_MIN_ABI_VERSION   3
-#define RDMA_USER_CM_MAX_ABI_VERSION   3
+#define RDMA_USER_CM_MAX_ABI_VERSION   4
 
 #define RDMA_MAX_PRIVATE_DATA          256
 
@@ -61,7 +61,9 @@ enum {
        UCMA_CMD_GET_EVENT,
        UCMA_CMD_GET_OPTION,
        UCMA_CMD_SET_OPTION,
-       UCMA_CMD_NOTIFY
+       UCMA_CMD_NOTIFY,
+       UCMA_CMD_JOIN_MCAST,
+       UCMA_CMD_LEAVE_MCAST
 };
 
 struct ucma_abi_cmd_hdr {
@@ -188,6 +190,13 @@ struct ucma_abi_notify {
        __u32 event;
 };
 
+struct ucma_abi_join_mcast {
+       __u64 response;         /* ucma_abi_create_id_resp */
+       __u64 uid;
+       struct sockaddr_in6 addr;
+       __u32 id;
+};
+
 struct ucma_abi_get_event {
        __u64 response;
 };

diff --git a/src/cma.c b/src/cma.c
index 043dab8816147a21299bda57bb2bdb439790a8f3..5cffa181247bf4b1b90c0552e9491788783a198a 100644 (file)
--- a/src/cma.c
+++ b/src/cma.c
@@ -115,12 +115,28 @@ struct cma_id_private {
        pthread_cond_t    cond;
        pthread_mutex_t   mut;
        uint32_t          handle;
+       struct cma_multicast *mc_list;
+};
+
+struct cma_multicast {
+       struct cma_multicast  *next;
+       struct cma_id_private *id_priv;
+       void            *context;
+       int             events_completed;
+       pthread_cond_t  cond;
+       uint32_t        handle;
+       union ibv_gid   mgid;
+       uint16_t        mlid;
+       struct sockaddr addr;
+       uint8_t         pad[sizeof(struct sockaddr_in6) -
+                           sizeof(struct sockaddr)];
 };
 
 struct cma_event {
        struct rdma_cm_event    event;
        uint8_t                 private_data[RDMA_MAX_PRIVATE_DATA];
        struct cma_id_private   *id_priv;
+       struct cma_multicast    *mc;
 };
 
 static struct cma_device *cma_dev_array;
@@ -907,6 +923,108 @@ int rdma_disconnect(struct rdma_cm_id *id)
        return 0;
 }
 
+int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
+                       void *context)
+{
+       struct ucma_abi_join_mcast *cmd;
+       struct ucma_abi_create_id_resp *resp;
+       struct cma_id_private *id_priv;
+       struct cma_multicast *mc, **pos;
+       void *msg;
+       int ret, size, addrlen;
+       
+       id_priv = container_of(id, struct cma_id_private, id);
+       addrlen = ucma_addrlen(addr);
+       if (!addrlen)
+               return -EINVAL;
+
+       mc = malloc(sizeof *mc);
+       if (!mc)
+               return -ENOMEM;
+
+       memset(mc, 0, sizeof *mc);
+       mc->context = context;
+       mc->id_priv = id_priv;
+       memcpy(&mc->addr, addr, addrlen);
+       if (pthread_cond_init(&id_priv->cond, NULL)) {
+               ret = -1;
+               goto err1;
+       }
+
+       pthread_mutex_lock(&id_priv->mut);
+       mc->next = id_priv->mc_list;
+       id_priv->mc_list = mc;
+       pthread_mutex_unlock(&id_priv->mut);
+
+       CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size);
+       cmd->id = id_priv->handle;
+       memcpy(&cmd->addr, addr, addrlen);
+       cmd->uid = (uintptr_t) mc;
+
+       ret = write(id->channel->fd, msg, size);
+       if (ret != size) {
+               ret = (ret > 0) ? -ENODATA : ret;
+               goto err2;
+       }
+
+       mc->handle = resp->id;
+       return 0;
+err2:
+       pthread_mutex_lock(&id_priv->mut);
+       for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next)
+               ;
+       *pos = mc->next;
+       pthread_mutex_unlock(&id_priv->mut);
+err1:
+       free(mc);
+       return ret;
+}
+
+int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
+{
+       struct ucma_abi_destroy_id *cmd;
+       struct ucma_abi_destroy_id_resp *resp;
+       struct cma_id_private *id_priv;
+       struct cma_multicast *mc, **pos;
+       void *msg;
+       int ret, size, addrlen;
+       
+       addrlen = ucma_addrlen(addr);
+       if (!addrlen)
+               return -EINVAL;
+
+       id_priv = container_of(id, struct cma_id_private, id);
+       pthread_mutex_lock(&id_priv->mut);
+       for (pos = &id_priv->mc_list; *pos; pos = &(*pos)->next)
+               if (!memcmp(&(*pos)->addr, addr, addrlen))
+                       break;
+
+       mc = *pos;
+       if (*pos)
+               *pos = mc->next;
+       pthread_mutex_unlock(&id_priv->mut);
+       if (!mc)
+               return -EADDRNOTAVAIL;
+
+       if (id->qp)
+               ibv_detach_mcast(id->qp, &mc->mgid, mc->mlid);
+       
+       CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_LEAVE_MCAST, size);
+       cmd->id = mc->handle;
+
+       ret = write(id->channel->fd, msg, size);
+       if (ret != size)
+               ret = (ret > 0) ? -ENODATA : ret;
+
+       pthread_mutex_lock(&id_priv->mut);
+       while (mc->events_completed < resp->events_reported)
+               pthread_cond_wait(&mc->cond, &id_priv->mut);
+       pthread_mutex_unlock(&id_priv->mut);
+
+       free(mc);
+       return ret;
+}
+
 static void ucma_complete_event(struct cma_id_private *id_priv)
 {
        pthread_mutex_lock(&id_priv->mut);
@@ -915,6 +1033,16 @@ static void ucma_complete_event(struct cma_id_private *id_priv)
        pthread_mutex_unlock(&id_priv->mut);
 }
 
+static void ucma_complete_mc_event(struct cma_multicast *mc)
+{
+       pthread_mutex_lock(&mc->id_priv->mut);
+       mc->events_completed++;
+       pthread_cond_signal(&mc->cond);
+       mc->id_priv->events_completed++;
+       pthread_cond_signal(&mc->id_priv->cond);
+       pthread_mutex_unlock(&mc->id_priv->mut);
+}
+
 int rdma_ack_cm_event(struct rdma_cm_event *event)
 {
        struct cma_event *evt;
@@ -924,7 +1052,10 @@ int rdma_ack_cm_event(struct rdma_cm_event *event)
 
        evt = container_of(event, struct cma_event, event);
 
-       ucma_complete_event(evt->id_priv);
+       if (evt->mc)
+               ucma_complete_mc_event(evt->mc);
+       else
+               ucma_complete_event(evt->id_priv);
        free(evt);
        return 0;
 }
@@ -999,6 +1130,18 @@ static int ucma_process_establish(struct rdma_cm_id *id)
        return ret;
 }
 
+static int ucma_process_join(struct cma_event *evt)
+{
+       evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
+       evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;
+
+       if (evt->id_priv->id.qp)
+               return ibv_attach_mcast(evt->id_priv->id.qp,
+                                       &evt->mc->mgid, evt->mc->mlid);
+       else
+               return 0;
+}
+
 static void ucma_copy_conn_event(struct cma_event *event,
                                 struct ucma_abi_conn_param *src)
 {
@@ -1139,6 +1282,23 @@ retry:
                evt->event.id = &evt->id_priv->id;
                ucma_copy_conn_event(evt, &resp->param.conn);
                break;
+       case RDMA_CM_EVENT_MULTICAST_JOIN:
+               evt->mc = (void *) (uintptr_t) resp->uid;
+               evt->id_priv = evt->mc->id_priv;
+               evt->event.id = &evt->id_priv->id;
+               ucma_copy_ud_event(evt, &resp->param.ud);
+               evt->event.param.ud.private_data = evt->mc->context;
+               evt->event.status = ucma_process_join(evt);
+               if (evt->event.status)
+                       evt->event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
+               break;
+       case RDMA_CM_EVENT_MULTICAST_ERROR:
+               evt->mc = (void *) (uintptr_t) resp->uid;
+               evt->id_priv = evt->mc->id_priv;
+               evt->event.id = &evt->id_priv->id;
+               evt->event.status = resp->status;
+               evt->event.param.ud.private_data = evt->mc->context;
+               break;
        default:
                evt->id_priv = (void *) (uintptr_t) resp->uid;
                evt->event.id = &evt->id_priv->id;

diff --git a/src/librdmacm.map b/src/librdmacm.map
index ed3f35a362fa9b320de09713094db0209ddb5d05..52986ca6646e41ca7f59ed5ab1a1781836ac640c 100644 (file)
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -1,4 +1,4 @@
-RDMACM_2.0 {
+RDMACM_1.0 {
        global:
                rdma_create_event_channel;
                rdma_destroy_event_channel;
@@ -19,6 +19,8 @@ RDMACM_2.0 {
                rdma_get_option;
                rdma_set_option;
                rdma_get_dst_attr;
+               rdma_join_multicast;
+               rdma_leave_multicast;
                rdma_get_devices;
                rdma_free_devices;
        local: *;