From: Sean Hefty Date: Tue, 5 Dec 2006 03:38:08 +0000 (-0800) Subject: Add multicast support to librdmacm X-Git-Tag: v1.0-rc1~11 X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=3e6ddeca5513d01ec4bfbff1c1698f5d4e22433d;p=~shefty%2Flibrdmacm.git Add multicast support to librdmacm Add support for multicast QPs through the rdma cm library. --- diff --git a/Makefile.am b/Makefile.am index 6a1f5795..d4caf796 100644 --- a/Makefile.am +++ b/Makefile.am @@ -18,16 +18,15 @@ endif src_librdmacm_la_SOURCES = src/cma.c src_librdmacm_la_LDFLAGS = -avoid-version $(rdmacm_version_script) -bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy -#examples/mckey +bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey examples_ucmatose_SOURCES = examples/cmatose.c examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la examples_rping_SOURCES = examples/rping.c examples_rping_LDADD = $(top_builddir)/src/librdmacm.la examples_udaddy_SOURCES = examples/udaddy.c examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la -#examples_mckey_SOURCES = examples/mckey.c -#examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la +examples_mckey_SOURCES = examples/mckey.c +examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la librdmacmincludedir = $(includedir)/rdma diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h index 711348ac..88a25b2e 100644 --- a/include/rdma/rdma_cma.h +++ b/include/rdma/rdma_cma.h @@ -55,7 +55,9 @@ enum rdma_cm_event_type { RDMA_CM_EVENT_REJECTED, RDMA_CM_EVENT_ESTABLISHED, RDMA_CM_EVENT_DISCONNECTED, - RDMA_CM_EVENT_DEVICE_REMOVAL + RDMA_CM_EVENT_DEVICE_REMOVAL, + RDMA_CM_EVENT_MULTICAST_JOIN, + RDMA_CM_EVENT_MULTICAST_ERROR }; enum rdma_port_space { @@ -279,6 +281,24 @@ int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event); */ int rdma_disconnect(struct rdma_cm_id *id); +/** + * rdma_join_multicast - Join the multicast group specified by the given + * address. + * @id: Communication identifier associated with the request. + * @addr: Multicast address identifying the group to join. + * @context: User-defined context associated with the join request. The + * context is returned to the user through the private_data field in + * the rdma_cm_event. + */ +int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, + void *context); + +/** + * rdma_leave_multicast - Leave the multicast group specified by the given + * address. + */ +int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr); + /** * rdma_get_cm_event - Retrieves the next pending communications event, * if no event is pending waits for an event. diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h index cd2d0aad..2ffcf929 100644 --- a/include/rdma/rdma_cma_abi.h +++ b/include/rdma/rdma_cma_abi.h @@ -41,7 +41,7 @@ */ #define RDMA_USER_CM_MIN_ABI_VERSION 3 -#define RDMA_USER_CM_MAX_ABI_VERSION 3 +#define RDMA_USER_CM_MAX_ABI_VERSION 4 #define RDMA_MAX_PRIVATE_DATA 256 @@ -61,7 +61,9 @@ enum { UCMA_CMD_GET_EVENT, UCMA_CMD_GET_OPTION, UCMA_CMD_SET_OPTION, - UCMA_CMD_NOTIFY + UCMA_CMD_NOTIFY, + UCMA_CMD_JOIN_MCAST, + UCMA_CMD_LEAVE_MCAST }; struct ucma_abi_cmd_hdr { @@ -188,6 +190,13 @@ struct ucma_abi_notify { __u32 event; }; +struct ucma_abi_join_mcast { + __u64 response; /* ucma_abi_create_id_resp */ + __u64 uid; + struct sockaddr_in6 addr; + __u32 id; +}; + struct ucma_abi_get_event { __u64 response; }; diff --git a/src/cma.c b/src/cma.c index 043dab88..5cffa181 100644 --- a/src/cma.c +++ b/src/cma.c @@ -115,12 +115,28 @@ struct cma_id_private { pthread_cond_t cond; pthread_mutex_t mut; uint32_t handle; + struct cma_multicast *mc_list; +}; + +struct cma_multicast { + struct cma_multicast *next; + struct cma_id_private *id_priv; + void *context; + int events_completed; + pthread_cond_t cond; + uint32_t handle; + union ibv_gid mgid; + uint16_t mlid; + struct sockaddr addr; + uint8_t pad[sizeof(struct sockaddr_in6) - + sizeof(struct sockaddr)]; }; struct cma_event { struct rdma_cm_event event; uint8_t private_data[RDMA_MAX_PRIVATE_DATA]; struct cma_id_private *id_priv; + struct cma_multicast *mc; }; static struct cma_device *cma_dev_array; @@ -907,6 +923,108 @@ int rdma_disconnect(struct rdma_cm_id *id) return 0; } +int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, + void *context) +{ + struct ucma_abi_join_mcast *cmd; + struct ucma_abi_create_id_resp *resp; + struct cma_id_private *id_priv; + struct cma_multicast *mc, **pos; + void *msg; + int ret, size, addrlen; + + id_priv = container_of(id, struct cma_id_private, id); + addrlen = ucma_addrlen(addr); + if (!addrlen) + return -EINVAL; + + mc = malloc(sizeof *mc); + if (!mc) + return -ENOMEM; + + memset(mc, 0, sizeof *mc); + mc->context = context; + mc->id_priv = id_priv; + memcpy(&mc->addr, addr, addrlen); + if (pthread_cond_init(&id_priv->cond, NULL)) { + ret = -1; + goto err1; + } + + pthread_mutex_lock(&id_priv->mut); + mc->next = id_priv->mc_list; + id_priv->mc_list = mc; + pthread_mutex_unlock(&id_priv->mut); + + CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size); + cmd->id = id_priv->handle; + memcpy(&cmd->addr, addr, addrlen); + cmd->uid = (uintptr_t) mc; + + ret = write(id->channel->fd, msg, size); + if (ret != size) { + ret = (ret > 0) ? -ENODATA : ret; + goto err2; + } + + mc->handle = resp->id; + return 0; +err2: + pthread_mutex_lock(&id_priv->mut); + for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next) + ; + *pos = mc->next; + pthread_mutex_unlock(&id_priv->mut); +err1: + free(mc); + return ret; +} + +int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) +{ + struct ucma_abi_destroy_id *cmd; + struct ucma_abi_destroy_id_resp *resp; + struct cma_id_private *id_priv; + struct cma_multicast *mc, **pos; + void *msg; + int ret, size, addrlen; + + addrlen = ucma_addrlen(addr); + if (!addrlen) + return -EINVAL; + + id_priv = container_of(id, struct cma_id_private, id); + pthread_mutex_lock(&id_priv->mut); + for (pos = &id_priv->mc_list; *pos; pos = &(*pos)->next) + if (!memcmp(&(*pos)->addr, addr, addrlen)) + break; + + mc = *pos; + if (*pos) + *pos = mc->next; + pthread_mutex_unlock(&id_priv->mut); + if (!mc) + return -EADDRNOTAVAIL; + + if (id->qp) + ibv_detach_mcast(id->qp, &mc->mgid, mc->mlid); + + CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_LEAVE_MCAST, size); + cmd->id = mc->handle; + + ret = write(id->channel->fd, msg, size); + if (ret != size) + ret = (ret > 0) ? -ENODATA : ret; + + pthread_mutex_lock(&id_priv->mut); + while (mc->events_completed < resp->events_reported) + pthread_cond_wait(&mc->cond, &id_priv->mut); + pthread_mutex_unlock(&id_priv->mut); + + free(mc); + return ret; +} + static void ucma_complete_event(struct cma_id_private *id_priv) { pthread_mutex_lock(&id_priv->mut); @@ -915,6 +1033,16 @@ static void ucma_complete_event(struct cma_id_private *id_priv) pthread_mutex_unlock(&id_priv->mut); } +static void ucma_complete_mc_event(struct cma_multicast *mc) +{ + pthread_mutex_lock(&mc->id_priv->mut); + mc->events_completed++; + pthread_cond_signal(&mc->cond); + mc->id_priv->events_completed++; + pthread_cond_signal(&mc->id_priv->cond); + pthread_mutex_unlock(&mc->id_priv->mut); +} + int rdma_ack_cm_event(struct rdma_cm_event *event) { struct cma_event *evt; @@ -924,7 +1052,10 @@ int rdma_ack_cm_event(struct rdma_cm_event *event) evt = container_of(event, struct cma_event, event); - ucma_complete_event(evt->id_priv); + if (evt->mc) + ucma_complete_mc_event(evt->mc); + else + ucma_complete_event(evt->id_priv); free(evt); return 0; } @@ -999,6 +1130,18 @@ static int ucma_process_establish(struct rdma_cm_id *id) return ret; } +static int ucma_process_join(struct cma_event *evt) +{ + evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid; + evt->mc->mlid = evt->event.param.ud.ah_attr.dlid; + + if (evt->id_priv->id.qp) + return ibv_attach_mcast(evt->id_priv->id.qp, + &evt->mc->mgid, evt->mc->mlid); + else + return 0; +} + static void ucma_copy_conn_event(struct cma_event *event, struct ucma_abi_conn_param *src) { @@ -1139,6 +1282,23 @@ retry: evt->event.id = &evt->id_priv->id; ucma_copy_conn_event(evt, &resp->param.conn); break; + case RDMA_CM_EVENT_MULTICAST_JOIN: + evt->mc = (void *) (uintptr_t) resp->uid; + evt->id_priv = evt->mc->id_priv; + evt->event.id = &evt->id_priv->id; + ucma_copy_ud_event(evt, &resp->param.ud); + evt->event.param.ud.private_data = evt->mc->context; + evt->event.status = ucma_process_join(evt); + if (evt->event.status) + evt->event.event = RDMA_CM_EVENT_MULTICAST_ERROR; + break; + case RDMA_CM_EVENT_MULTICAST_ERROR: + evt->mc = (void *) (uintptr_t) resp->uid; + evt->id_priv = evt->mc->id_priv; + evt->event.id = &evt->id_priv->id; + evt->event.status = resp->status; + evt->event.param.ud.private_data = evt->mc->context; + break; default: evt->id_priv = (void *) (uintptr_t) resp->uid; evt->event.id = &evt->id_priv->id; diff --git a/src/librdmacm.map b/src/librdmacm.map index ed3f35a3..52986ca6 100644 --- a/src/librdmacm.map +++ b/src/librdmacm.map @@ -1,4 +1,4 @@ -RDMACM_2.0 { +RDMACM_1.0 { global: rdma_create_event_channel; rdma_destroy_event_channel; @@ -19,6 +19,8 @@ RDMACM_2.0 { rdma_get_option; rdma_set_option; rdma_get_dst_attr; + rdma_join_multicast; + rdma_leave_multicast; rdma_get_devices; rdma_free_devices; local: *;