From 6f640ff22c3d6680e6413ad24ff6ff262ed688d9 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Tue, 9 May 2006 18:25:10 +0000 Subject: [PATCH] r7019: Introduce event channels. Event channels allow the application to direct how RDMA CM events are reported, by directing events on different rdma_cm_id's to different event channels. These are similar to completion channels used in verbs. Signed-off-by: Sean Hefty --- examples/cmatose.c | 18 +++++-- examples/rping.c | 17 +++++-- include/rdma/rdma_cma.h | 38 +++++++++++++-- src/cma.c | 102 ++++++++++++++++++++++------------------ src/librdmacm.map | 3 +- 5 files changed, 120 insertions(+), 58 deletions(-) diff --git a/examples/cmatose.c b/examples/cmatose.c index 218a8feb..92d1251e 100644 --- a/examples/cmatose.c +++ b/examples/cmatose.c @@ -69,6 +69,7 @@ struct cmatest_node { }; struct cmatest { + struct rdma_event_channel *channel; struct cmatest_node *nodes; int conn_index; int connects_left; @@ -375,7 +376,8 @@ static int alloc_nodes(void) for (i = 0; i < connections; i++) { test.nodes[i].id = i; if (!is_server) { - ret = rdma_create_id(&test.nodes[i].cma_id, + ret = rdma_create_id(test.channel, + &test.nodes[i].cma_id, &test.nodes[i]); if (ret) goto err; @@ -424,7 +426,7 @@ static void connect_events(void) int err = 0; while (test.connects_left && !err) { - err = rdma_get_cm_event(&event); + err = rdma_get_cm_event(test.channel, &event); if (!err) { cma_handler(event->id, event); rdma_ack_cm_event(event); @@ -438,7 +440,7 @@ static void disconnect_events(void) int err = 0; while (test.disconnects_left && !err) { - err = rdma_get_cm_event(&event); + err = rdma_get_cm_event(test.channel, &event); if (!err) { cma_handler(event->id, event); rdma_ack_cm_event(event); @@ -452,7 +454,7 @@ static void run_server(void) int i, ret; printf("cmatose: starting server\n"); - ret = rdma_create_id(&listen_id, &test); + ret = rdma_create_id(test.channel, &listen_id, &test); if (ret) { printf("cmatose: listen request failed\n"); return; @@ -582,6 +584,13 @@ int main(int argc, char **argv) test.src_addr = (struct sockaddr *) &test.src_in; test.connects_left = connections; test.disconnects_left = connections; + + test.channel = rdma_create_event_channel(); + if (!test.channel) { + printf("failed to create event channel\n"); + exit(1); + } + if (alloc_nodes()) exit(1); @@ -592,5 +601,6 @@ int main(int argc, char **argv) printf("test complete\n"); destroy_nodes(); + rdma_destroy_event_channel(test.channel); return 0; } diff --git a/examples/rping.c b/examples/rping.c index a07eb83f..b047cc34 100644 --- a/examples/rping.c +++ b/examples/rping.c @@ -142,6 +142,7 @@ struct rping_cb { /* CM stuff */ pthread_t cmthread; + struct rdma_event_channel *cm_channel; struct rdma_cm_id *cm_id; /* connection on client side,*/ /* listener on service side. */ struct rdma_cm_id *child_cm_id; /* connection on server side */ @@ -534,11 +535,12 @@ err1: static void *cm_thread(void *arg) { + struct rping_cb *cb = arg; struct rdma_cm_event *event; int ret; while (1) { - ret = rdma_get_cm_event(&event); + ret = rdma_get_cm_event(cb->cm_channel, &event); if (ret) { fprintf(stderr, "rdma_get_cm_event err %d\n", ret); exit(ret); @@ -1019,11 +1021,18 @@ int main(int argc, char *argv[]) goto out; } - ret = rdma_create_id(&cb->cm_id, cb); + cb->cm_channel = rdma_create_event_channel(); + if (!cb->cm_channel) { + ret = errno; + fprintf(stderr, "rdma_create_event_channel error %d\n", ret); + goto out; + } + + ret = rdma_create_id(cb->cm_channel, &cb->cm_id, cb); if (ret) { ret = errno; fprintf(stderr, "rdma_create_id error %d\n", ret); - goto out; + goto out2; } DEBUG_LOG("created cm_id %p\n", cb->cm_id); @@ -1036,6 +1045,8 @@ int main(int argc, char *argv[]) DEBUG_LOG("destroy cm_id %p\n", cb->cm_id); rdma_destroy_id(cb->cm_id); +out2: + rdma_destroy_event_channel(cb->cm_channel); out: free(cb); return ret; diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h index 2ab0a854..2f415ef0 100644 --- a/include/rdma/rdma_cma.h +++ b/include/rdma/rdma_cma.h @@ -85,8 +85,13 @@ struct rdma_route { int num_paths; }; +struct rdma_event_channel { + int fd; +}; + struct rdma_cm_id { struct ibv_context *verbs; + struct rdma_event_channel *channel; void *context; struct ibv_qp *qp; struct rdma_route route; @@ -102,8 +107,33 @@ struct rdma_cm_event { uint8_t private_data_len; }; -int rdma_create_id(struct rdma_cm_id **id, void *context); +/** + * rdma_create_event_channel - Open a channel used to report communication + * events. + */ +struct rdma_event_channel *rdma_create_event_channel(); + +/** + * rdma_destroy_event_channel - Close the event communication channel. + * @channel: The communication channel to destroy. + */ +int rdma_destroy_event_channel(struct rdma_event_channel *channel); +/** + * rdma_create_id - Allocate a communication identifier. + * @channel: The communication channel that events associated with the + * allocated rdma_cm_id will be reported on. + * @id: A reference where the allocated communication identifier will be + * returned. + * @context: User specified context associated with the rdma_cm_id. + */ +int rdma_create_id(struct rdma_event_channel *channel, + struct rdma_cm_id **id, void *context); + +/** + * rdma_destroy_id - Release a communication identifier. + * @id: The communication identifier to destroy. + */ int rdma_destroy_id(struct rdma_cm_id *id); /** @@ -209,6 +239,7 @@ int rdma_disconnect(struct rdma_cm_id *id); /** * rdma_get_cm_event - Retrieves the next pending communications event, * if no event is pending waits for an event. + * @channel: Event channel to check for events. * @event: Allocated information about the next communication event. * Event should be freed using rdma_ack_cm_event() * @@ -216,7 +247,8 @@ int rdma_disconnect(struct rdma_cm_id *id); * in the allocation of a new @rdma_cm_id. * Clients are responsible for destroying the new @rdma_cm_id. */ -int rdma_get_cm_event(struct rdma_cm_event **event); +int rdma_get_cm_event(struct rdma_event_channel *channel, + struct rdma_cm_event **event); /** * rdma_ack_cm_event - Free a communications event. @@ -228,8 +260,6 @@ int rdma_get_cm_event(struct rdma_cm_event **event); */ int rdma_ack_cm_event(struct rdma_cm_event *event); -int rdma_get_fd(void); - /** * rdma_get_option - Retrieve options for an rdma_cm_id. * @id: Communication identifier to retrieve option for. diff --git a/src/cma.c b/src/cma.c index b51b56a4..26596a39 100644 --- a/src/cma.c +++ b/src/cma.c @@ -120,7 +120,6 @@ static struct dlist *cma_dev_list; static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER; static int ucma_initialized; static int abi_ver; -int cma_fd; #define container_of(ptr, type, field) \ ((type *) ((void *)ptr - offsetof(type, field))) @@ -136,9 +135,6 @@ static void ucma_cleanup(void) dlist_destroy(cma_dev_list); cma_dev_list = NULL; } - - if (cma_fd > 0) - close(cma_fd); } static int check_abi_version(void) @@ -176,13 +172,6 @@ static int ucma_init(void) if (ucma_initialized) goto out; - cma_fd = open("/dev/infiniband/rdma_cm", O_RDWR); - if (cma_fd < 0) { - printf("CMA: unable to open /dev/infiniband/rdma_cm\n"); - ret = -ENOENT; - goto err; - } - ret = check_abi_version(); if (ret) goto err; @@ -241,6 +230,34 @@ static void __attribute__((destructor)) rdma_cma_fini(void) ucma_cleanup(); } +struct rdma_event_channel *rdma_create_event_channel() +{ + struct rdma_event_channel *channel; + + if (!ucma_initialized && ucma_init()) + return NULL; + + channel = malloc(sizeof *channel); + if (!channel) + return NULL; + + channel->fd = open("/dev/infiniband/rdma_cm", O_RDWR); + if (channel->fd < 0) { + printf("CMA: unable to open /dev/infiniband/rdma_cm\n"); + goto err; + } + return channel; +err: + free(channel); + return NULL; +} + +int rdma_destroy_event_channel(struct rdma_event_channel *channel) +{ + close(channel->fd); + free(channel); +} + static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid) { struct cma_device *cma_dev; @@ -264,7 +281,8 @@ static void ucma_free_id(struct cma_id_private *id_priv) free(id_priv); } -static struct cma_id_private *ucma_alloc_id(void *context) +static struct cma_id_private *ucma_alloc_id(struct rdma_event_channel *channel, + void *context) { struct cma_id_private *id_priv; @@ -274,6 +292,7 @@ static struct cma_id_private *ucma_alloc_id(void *context) memset(id_priv, 0, sizeof *id_priv); id_priv->id.context = context; + id_priv->id.channel = channel; pthread_mutex_init(&id_priv->mut, NULL); if (pthread_cond_init(&id_priv->cond, NULL)) goto err; @@ -284,7 +303,8 @@ err: ucma_free_id(id_priv); return NULL; } -int rdma_create_id(struct rdma_cm_id **id, void *context) +int rdma_create_id(struct rdma_event_channel *channel, + struct rdma_cm_id **id, void *context) { struct ucma_abi_create_id_resp *resp; struct ucma_abi_create_id *cmd; @@ -296,14 +316,14 @@ int rdma_create_id(struct rdma_cm_id **id, void *context) if (ret) return ret; - id_priv = ucma_alloc_id(context); + id_priv = ucma_alloc_id(channel, context); if (!id_priv) return -ENOMEM; CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size); cmd->uid = (uintptr_t) id_priv; - ret = write(cma_fd, msg, size); + ret = write(channel->fd, msg, size); if (ret != size) goto err; @@ -315,7 +335,7 @@ err: ucma_free_id(id_priv); return ret; } -static int ucma_destroy_kern_id(uint32_t handle) +static int ucma_destroy_kern_id(int fd, uint32_t handle) { struct ucma_abi_destroy_id_resp *resp; struct ucma_abi_destroy_id *cmd; @@ -325,7 +345,7 @@ static int ucma_destroy_kern_id(uint32_t handle) CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_DESTROY_ID, size); cmd->id = handle; - ret = write(cma_fd, msg, size); + ret = write(fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -338,7 +358,7 @@ int rdma_destroy_id(struct rdma_cm_id *id) int ret; id_priv = container_of(id, struct cma_id_private, id); - ret = ucma_destroy_kern_id(id_priv->handle); + ret = ucma_destroy_kern_id(id->channel->fd, id_priv->handle); if (ret < 0) return ret; @@ -378,7 +398,7 @@ static int ucma_query_route(struct rdma_cm_id *id) id_priv = container_of(id, struct cma_id_private, id); cmd->id = id_priv->handle; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -430,7 +450,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) cmd->id = id_priv->handle; memcpy(&cmd->addr, addr, addrlen); - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -462,7 +482,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, memcpy(&cmd->dst_addr, dst_addr, daddrlen); cmd->timeout_ms = timeout_ms; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -482,7 +502,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) cmd->id = id_priv->handle; cmd->timeout_ms = timeout_ms; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -503,7 +523,7 @@ static int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr, cmd->id = id_priv->handle; cmd->qp_state = qp_attr->qp_state; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -663,7 +683,7 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) cmd->id = id_priv->handle; ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, id->qp); - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -682,7 +702,7 @@ int rdma_listen(struct rdma_cm_id *id, int backlog) cmd->id = id_priv->handle; cmd->backlog = backlog; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -706,7 +726,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) cmd->uid = (uintptr_t) id_priv; ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, id->qp); - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) { ucma_modify_qp_err(id); return (ret > 0) ? -ENODATA : ret; @@ -733,7 +753,7 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data, } else cmd->private_data_len = 0; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -755,7 +775,7 @@ int rdma_disconnect(struct rdma_cm_id *id) id_priv = container_of(id, struct cma_id_private, id); cmd->id = id_priv->handle; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -806,9 +826,9 @@ static int ucma_process_conn_req(struct rdma_cm_event *event, int ret; listen_id_priv = container_of(event->id, struct cma_id_private, id); - id_priv = ucma_alloc_id(event->id->context); + id_priv = ucma_alloc_id(event->id->channel, event->id->context); if (!id_priv) { - ucma_destroy_kern_id(handle); + ucma_destroy_kern_id(event->id->channel->fd, handle); ret = -ENOMEM; goto err; } @@ -846,7 +866,7 @@ static int ucma_process_conn_resp(struct cma_id_private *id_priv) CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size); cmd->id = id_priv->handle; - ret = write(cma_fd, msg, size); + ret = write(id_priv->id.channel->fd, msg, size); if (ret != size) { ret = (ret > 0) ? -ENODATA : ret; goto err; @@ -869,7 +889,8 @@ static int ucma_process_establish(struct rdma_cm_id *id) return ret; } -int rdma_get_cm_event(struct rdma_cm_event **event) +int rdma_get_cm_event(struct rdma_event_channel *channel, + struct rdma_cm_event **event) { struct ucma_abi_event_resp *resp; struct ucma_abi_get_event *cmd; @@ -891,7 +912,7 @@ int rdma_get_cm_event(struct rdma_cm_event **event) retry: CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size); - ret = write(cma_fd, msg, size); + ret = write(channel->fd, msg, size); if (ret != size) { ret = (ret > 0) ? -ENODATA : ret; goto err; @@ -953,17 +974,6 @@ err: return ret; } -int rdma_get_fd(void) -{ - int ret; - - ret = ucma_initialized ? 0 : ucma_init(); - if (ret) - return ret; - - return cma_fd; -} - int rdma_get_option(struct rdma_cm_id *id, int level, int optname, void *optval, size_t *optlen) { @@ -981,7 +991,7 @@ int rdma_get_option(struct rdma_cm_id *id, int level, int optname, cmd->optname = optname; cmd->optlen = *optlen; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; @@ -1005,7 +1015,7 @@ int rdma_set_option(struct rdma_cm_id *id, int level, int optname, cmd->optname = optname; cmd->optlen = optlen; - ret = write(cma_fd, msg, size); + ret = write(id->channel->fd, msg, size); if (ret != size) return (ret > 0) ? -ENODATA : ret; diff --git a/src/librdmacm.map b/src/librdmacm.map index d14dd94f..23617758 100644 --- a/src/librdmacm.map +++ b/src/librdmacm.map @@ -1,5 +1,7 @@ RDMACM_1.0 { global: + rdma_create_event_channel; + rdma_destroy_event_channel; rdma_create_id; rdma_destroy_id; rdma_bind_addr; @@ -14,7 +16,6 @@ RDMACM_1.0 { rdma_disconnect; rdma_get_cm_event; rdma_ack_cm_event; - rdma_get_fd; rdma_get_option; rdma_set_option; local: *; -- 2.41.0