From 470e35344eca278ab5efdd0b1d3a86cf083812ca Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Mon, 4 Dec 2006 19:17:44 -0800 Subject: [PATCH] Update librdmacm to support ABI version 3 rdma_ucm ABI 3 adds support for reporting connection information with connect events and UDP port space support. Update the librdmacm to take advantage of these features, and update test programs as well. --- Makefile.am | 11 +- examples/cmatose.c | 117 ++++++----- examples/mckey.c | 81 +++++--- examples/udaddy.c | 138 +++++++------ include/rdma/rdma_cma.h | 130 +++++------- include/rdma/rdma_cma_abi.h | 75 ++----- include/rdma/rdma_cma_ib.h | 68 ------ src/cma.c | 402 ++++++++++++------------------------ src/librdmacm.map | 4 +- 9 files changed, 409 insertions(+), 617 deletions(-) delete mode 100644 include/rdma/rdma_cma_ib.h diff --git a/Makefile.am b/Makefile.am index 8fc24a25..6a1f5795 100644 --- a/Makefile.am +++ b/Makefile.am @@ -18,25 +18,24 @@ endif src_librdmacm_la_SOURCES = src/cma.c src_librdmacm_la_LDFLAGS = -avoid-version $(rdmacm_version_script) -bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey +bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy +#examples/mckey examples_ucmatose_SOURCES = examples/cmatose.c examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la examples_rping_SOURCES = examples/rping.c examples_rping_LDADD = $(top_builddir)/src/librdmacm.la examples_udaddy_SOURCES = examples/udaddy.c examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la -examples_mckey_SOURCES = examples/mckey.c -examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la +#examples_mckey_SOURCES = examples/mckey.c +#examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la librdmacmincludedir = $(includedir)/rdma librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \ - include/rdma/rdma_cma.h \ - include/rdma/rdma_cma_ib.h + include/rdma/rdma_cma.h EXTRA_DIST = include/rdma/rdma_cma_abi.h \ include/rdma/rdma_cma.h \ - include/rdma/rdma_cma_ib.h \ src/librdmacm.map \ librdmacm.spec.in diff --git a/examples/cmatose.c b/examples/cmatose.c index ca464020..4479fd48 100644 --- a/examples/cmatose.c +++ b/examples/cmatose.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005 Intel Corporation. All rights reserved. + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -41,6 +41,7 @@ #include #include #include +#include #include @@ -52,12 +53,6 @@ static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); } static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); } #endif -/* - * To execute: - * Server: rdma_cmatose - * Client: rdma_cmatose - */ - struct cmatest_node { int id; struct rdma_cm_id *cma_id; @@ -85,7 +80,8 @@ static struct cmatest test; static int connections = 1; static int message_size = 100; static int message_count = 10; -static int is_server; +static char *dst_addr; +static char *src_addr; static int create_message(struct cmatest_node *node) { @@ -377,7 +373,7 @@ static int alloc_nodes(void) for (i = 0; i < connections; i++) { test.nodes[i].id = i; - if (!is_server) { + if (dst_addr) { ret = rdma_create_id(test.channel, &test.nodes[i].cma_id, &test.nodes[i], RDMA_PS_TCP); @@ -460,6 +456,28 @@ static int disconnect_events(void) return ret; } +static int get_addr(char *dst, struct sockaddr_in *addr) +{ + struct addrinfo *res; + int ret; + + ret = getaddrinfo(dst, NULL, NULL, &res); + if (ret) { + printf("getaddrinfo failed - invalid hostname or IP address\n"); + return ret; + } + + if (res->ai_family != PF_INET) { + ret = -1; + goto out; + } + + *addr = *(struct sockaddr_in *) res->ai_addr; +out: + freeaddrinfo(res); + return ret; +} + static int run_server(void) { struct rdma_cm_id *listen_id; @@ -472,12 +490,18 @@ static int run_server(void) return ret; } - test.src_in.sin_family = PF_INET; + if (src_addr) { + ret = get_addr(src_addr, &test.src_in); + if (ret) + goto out; + } else + test.src_in.sin_family = PF_INET; + test.src_in.sin_port = 7471; ret = rdma_bind_addr(listen_id, test.src_addr); if (ret) { printf("cmatose: bind address failed: %d\n", ret); - return ret; + goto out; } ret = rdma_listen(listen_id, 0); @@ -528,40 +552,18 @@ out: return ret; } -static int get_addr(char *dst, struct sockaddr_in *addr) -{ - struct addrinfo *res; - int ret; - - ret = getaddrinfo(dst, NULL, NULL, &res); - if (ret) { - printf("getaddrinfo failed - invalid hostname or IP address\n"); - return ret; - } - - if (res->ai_family != PF_INET) { - ret = -1; - goto out; - } - - *addr = *(struct sockaddr_in *) res->ai_addr; -out: - freeaddrinfo(res); - return ret; -} - -static int run_client(char *dst, char *src) +static int run_client(void) { int i, ret, ret2; printf("cmatose: starting client\n"); - if (src) { - ret = get_addr(src, &test.src_in); + if (src_addr) { + ret = get_addr(src_addr, &test.src_in); if (ret) return ret; } - ret = get_addr(dst, &test.dst_in); + ret = get_addr(dst_addr, &test.dst_in); if (ret) return ret; @@ -570,7 +572,7 @@ static int run_client(char *dst, char *src) printf("cmatose: connecting\n"); for (i = 0; i < connections; i++) { ret = rdma_resolve_addr(test.nodes[i].cma_id, - src ? test.src_addr : NULL, + src_addr ? test.src_addr : NULL, test.dst_addr, 2000); if (ret) { printf("cmatose: failure getting addr: %d\n", ret); @@ -597,7 +599,6 @@ static int run_client(char *dst, char *src) } printf("data transfers complete\n"); - } ret = 0; @@ -611,13 +612,35 @@ out: int main(int argc, char **argv) { - int ret; + int op, ret; - if (argc > 3) { - printf("usage: %s [server_addr [src_addr]]\n", argv[0]); - exit(1); + while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) { + switch (op) { + case 's': + dst_addr = optarg; + break; + case 'b': + src_addr = optarg; + break; + case 'c': + connections = atoi(optarg); + break; + case 'C': + message_count = atoi(optarg); + break; + case 'S': + message_size = atoi(optarg); + break; + default: + printf("usage: %s\n", argv[0]); + printf("\t[-s server_address]\n"); + printf("\t[-b bind_address]\n"); + printf("\t[-c connections]\n"); + printf("\t[-C message_count]\n"); + printf("\t[-S message_size]\n"); + exit(1); + } } - is_server = (argc == 1); test.dst_addr = (struct sockaddr *) &test.dst_in; test.src_addr = (struct sockaddr *) &test.src_in; @@ -633,10 +656,10 @@ int main(int argc, char **argv) if (alloc_nodes()) exit(1); - if (is_server) - ret = run_server(); + if (dst_addr) + ret = run_client(); else - ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL); + ret = run_server(); printf("test complete\n"); destroy_nodes(); diff --git a/examples/mckey.c b/examples/mckey.c index 96250dec..31dbac67 100644 --- a/examples/mckey.c +++ b/examples/mckey.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005 Intel Corporation. All rights reserved. + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -42,9 +42,9 @@ #include #include #include +#include #include -#include struct cmatest_node { int id; @@ -76,6 +76,8 @@ static int connections = 1; static int message_size = 100; static int message_count = 10; static int is_sender; +static char *dst_addr; +static char *src_addr; static int create_message(struct cmatest_node *node) { @@ -239,19 +241,12 @@ err: return ret; } -static int join_handler(struct cmatest_node *node) +static int join_handler(struct cmatest_node *node, + struct rdma_ud_param *param) { - struct ibv_ah_attr ah_attr; - int ret; - - ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr, - &node->remote_qpn, &node->remote_qkey); - if (ret) { - printf("mckey: failure getting destination attributes\n"); - goto err; - } - - node->ah = ibv_create_ah(node->pd, &ah_attr); + node->remote_qpn = param->qp_num; + node->remote_qkey = param->qkey; + node->ah = ibv_create_ah(node->pd, ¶m->ah_attr); if (!node->ah) { printf("mckey: failure creating address handle\n"); goto err; @@ -262,7 +257,7 @@ static int join_handler(struct cmatest_node *node) return 0; err: connect_error(); - return ret; + return -1; } static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) @@ -274,7 +269,7 @@ static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) ret = addr_handler(cma_id->context); break; case RDMA_CM_EVENT_MULTICAST_JOIN: - ret = join_handler(cma_id->context); + ret = join_handler(cma_id->context, &event->param.ud); break; case RDMA_CM_EVENT_ADDR_ERROR: case RDMA_CM_EVENT_ROUTE_ERROR: @@ -411,18 +406,21 @@ out: return ret; } -static int run(char *dst, char *src) +static int run(void) { int i, ret; - printf("mckey: starting client\n"); - if (src) { - ret = get_addr(src, &test.src_in); + if (is_sender) + printf("mckey: starting client\n"); + else + printf("mckey: starting server\n"); + if (src_addr) { + ret = get_addr(src_addr, &test.src_in); if (ret) return ret; } - ret = get_addr(dst, &test.dst_in); + ret = get_addr(dst_addr, &test.dst_in); if (ret) return ret; @@ -431,7 +429,7 @@ static int run(char *dst, char *src) printf("mckey: joining\n"); for (i = 0; i < connections; i++) { ret = rdma_resolve_addr(test.nodes[i].cma_id, - src ? test.src_addr : NULL, + src_addr ? test.src_addr : NULL, test.dst_addr, 2000); if (ret) { printf("mckey: failure getting addr: %d\n", ret); @@ -472,14 +470,39 @@ out: int main(int argc, char **argv) { - int ret; + int op, ret; - if (argc < 3 || argc > 4) { - printf("usage: %s {s[end] | r[ecv]} mcast_addr [bind_addr]]\n", - argv[0]); - exit(1); + while ((op = getopt(argc, argv, "m:sb:c:C:S:")) != -1) { + switch (op) { + case 'm': + dst_addr = optarg; + break; + case 's': + is_sender = 1; + break; + case 'b': + src_addr = optarg; + break; + case 'c': + connections = atoi(optarg); + break; + case 'C': + message_count = atoi(optarg); + break; + case 'S': + message_size = atoi(optarg); + break; + default: + printf("usage: %s\n", argv[0]); + printf("\t-m multicast_address\n"); + printf("\t[-s(ender)]\n"); + printf("\t[-b bind_address]\n"); + printf("\t[-c connections]\n"); + printf("\t[-C message_count]\n"); + printf("\t[-S message_size]\n"); + exit(1); + } } - is_sender = (argv[1][0] == 's'); test.dst_addr = (struct sockaddr *) &test.dst_in; test.src_addr = (struct sockaddr *) &test.src_in; @@ -494,7 +517,7 @@ int main(int argc, char **argv) if (alloc_nodes()) exit(1); - ret = run(argv[2], (argc == 4) ? argv[3] : NULL); + ret = run(); printf("test complete\n"); destroy_nodes(); diff --git a/examples/udaddy.c b/examples/udaddy.c index f6896c2f..ab9ace69 100644 --- a/examples/udaddy.c +++ b/examples/udaddy.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005 Intel Corporation. All rights reserved. + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -41,15 +41,9 @@ #include #include #include +#include #include -#include - -/* - * To execute: - * Server: udaddy - * Client: udaddy [server_addr [src_addr]] - */ struct cmatest_node { int id; @@ -80,7 +74,8 @@ static struct cmatest test; static int connections = 1; static int message_size = 100; static int message_count = 10; -static int is_server; +static char *dst_addr; +static char *src_addr; static int create_message(struct cmatest_node *node) { @@ -246,7 +241,6 @@ static int route_handler(struct cmatest_node *node) memset(&conn_param, 0, sizeof conn_param); conn_param.qp_num = node->cma_id->qp->qp_num; - conn_param.qp_type = node->cma_id->qp->qp_type; conn_param.retry_count = 5; ret = rdma_connect(node->cma_id, &conn_param); if (ret) { @@ -284,7 +278,6 @@ static int connect_handler(struct rdma_cm_id *cma_id) memset(&conn_param, 0, sizeof conn_param); conn_param.qp_num = node->cma_id->qp->qp_num; - conn_param.qp_type = node->cma_id->qp->qp_type; ret = rdma_accept(node->cma_id, &conn_param); if (ret) { printf("udaddy: failure accepting: %d\n", ret); @@ -303,19 +296,12 @@ err1: return ret; } -static int resolved_handler(struct cmatest_node *node) +static int resolved_handler(struct cmatest_node *node, + struct rdma_cm_event *event) { - struct ibv_ah_attr ah_attr; - int ret; - - ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr, - &node->remote_qpn, &node->remote_qkey); - if (ret) { - printf("udaddy: failure getting destination attributes\n"); - goto err; - } - - node->ah = ibv_create_ah(node->pd, &ah_attr); + node->remote_qpn = event->param.ud.qp_num; + node->remote_qkey = event->param.ud.qkey; + node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr); if (!node->ah) { printf("udaddy: failure creating address handle\n"); goto err; @@ -326,7 +312,7 @@ static int resolved_handler(struct cmatest_node *node) return 0; err: connect_error(); - return ret; + return -1; } static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) @@ -344,7 +330,7 @@ static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) ret = connect_handler(cma_id); break; case RDMA_CM_EVENT_ESTABLISHED: - ret = resolved_handler(cma_id->context); + ret = resolved_handler(cma_id->context, event); break; case RDMA_CM_EVENT_ADDR_ERROR: case RDMA_CM_EVENT_ROUTE_ERROR: @@ -404,7 +390,7 @@ static int alloc_nodes(void) for (i = 0; i < connections; i++) { test.nodes[i].id = i; - if (!is_server) { + if (dst_addr) { ret = rdma_create_id(test.channel, &test.nodes[i].cma_id, &test.nodes[i], RDMA_PS_UDP); @@ -475,6 +461,28 @@ static int connect_events(void) return ret; } +static int get_addr(char *dst, struct sockaddr_in *addr) +{ + struct addrinfo *res; + int ret; + + ret = getaddrinfo(dst, NULL, NULL, &res); + if (ret) { + printf("getaddrinfo failed - invalid hostname or IP address\n"); + return ret; + } + + if (res->ai_family != PF_INET) { + ret = -1; + goto out; + } + + *addr = *(struct sockaddr_in *) res->ai_addr; +out: + freeaddrinfo(res); + return ret; +} + static int run_server(void) { struct rdma_cm_id *listen_id; @@ -487,7 +495,13 @@ static int run_server(void) return ret; } - test.src_in.sin_family = PF_INET; + if (src_addr) { + ret = get_addr(src_addr, &test.src_in); + if (ret) + goto out; + } else + test.src_in.sin_family = PF_INET; + test.src_in.sin_port = 7174; ret = rdma_bind_addr(listen_id, test.src_addr); if (ret) { @@ -526,40 +540,18 @@ out: return ret; } -static int get_addr(char *dst, struct sockaddr_in *addr) -{ - struct addrinfo *res; - int ret; - - ret = getaddrinfo(dst, NULL, NULL, &res); - if (ret) { - printf("getaddrinfo failed - invalid hostname or IP address\n"); - return ret; - } - - if (res->ai_family != PF_INET) { - ret = -1; - goto out; - } - - *addr = *(struct sockaddr_in *) res->ai_addr; -out: - freeaddrinfo(res); - return ret; -} - -static int run_client(char *dst, char *src) +static int run_client(void) { int i, ret; printf("udaddy: starting client\n"); - if (src) { - ret = get_addr(src, &test.src_in); + if (src_addr) { + ret = get_addr(src_addr, &test.src_in); if (ret) return ret; } - ret = get_addr(dst, &test.dst_in); + ret = get_addr(dst_addr, &test.dst_in); if (ret) return ret; @@ -568,7 +560,7 @@ static int run_client(char *dst, char *src) printf("udaddy: connecting\n"); for (i = 0; i < connections; i++) { ret = rdma_resolve_addr(test.nodes[i].cma_id, - src ? test.src_addr : NULL, + src_addr ? test.src_addr : NULL, test.dst_addr, 2000); if (ret) { printf("udaddy: failure getting addr: %d\n", ret); @@ -601,13 +593,35 @@ out: int main(int argc, char **argv) { - int ret; + int op, ret; - if (argc > 3) { - printf("usage: %s [server_addr [src_addr]]\n", argv[0]); - exit(1); + while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) { + switch (op) { + case 's': + dst_addr = optarg; + break; + case 'b': + src_addr = optarg; + break; + case 'c': + connections = atoi(optarg); + break; + case 'C': + message_count = atoi(optarg); + break; + case 'S': + message_size = atoi(optarg); + break; + default: + printf("usage: %s\n", argv[0]); + printf("\t[-s server_address]\n"); + printf("\t[-b bind_address]\n"); + printf("\t[-c connections]\n"); + printf("\t[-C message_count]\n"); + printf("\t[-S message_size]\n"); + exit(1); + } } - is_server = (argc == 1); test.dst_addr = (struct sockaddr *) &test.dst_in; test.src_addr = (struct sockaddr *) &test.src_in; @@ -622,10 +636,10 @@ int main(int argc, char **argv) if (alloc_nodes()) exit(1); - if (is_server) - ret = run_server(); + if (dst_addr) + ret = run_client(); else - ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL); + ret = run_server(); printf("test complete\n"); destroy_nodes(); diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h index 6b20b122..711348ac 100644 --- a/include/rdma/rdma_cma.h +++ b/include/rdma/rdma_cma.h @@ -55,9 +55,7 @@ enum rdma_cm_event_type { RDMA_CM_EVENT_REJECTED, RDMA_CM_EVENT_ESTABLISHED, RDMA_CM_EVENT_DISCONNECTED, - RDMA_CM_EVENT_DEVICE_REMOVAL, - RDMA_CM_EVENT_MULTICAST_JOIN, - RDMA_CM_EVENT_MULTICAST_ERROR + RDMA_CM_EVENT_DEVICE_REMOVAL }; enum rdma_port_space { @@ -65,11 +63,11 @@ enum rdma_port_space { RDMA_PS_UDP = 0x0111, }; -/* Protocol levels for get/set options. */ -enum { - RDMA_PROTO_IP = 0, - RDMA_PROTO_IB = 1, -}; +/* + * Global qkey value for all UD QPs and multicast groups created via the + * RDMA CM. + */ +#define RDMA_UD_QKEY 0x01234567 struct ib_addr { union ibv_gid sgid; @@ -78,8 +76,12 @@ struct ib_addr { }; struct rdma_addr { - struct sockaddr_in6 src_addr; - struct sockaddr_in6 dst_addr; + struct sockaddr src_addr; + uint8_t src_pad[sizeof(struct sockaddr_storage) - + sizeof(struct sockaddr)]; + struct sockaddr dst_addr; + uint8_t dst_pad[sizeof(struct sockaddr_storage) - + sizeof(struct sockaddr)]; union { struct ib_addr ibaddr; } addr; @@ -105,11 +107,25 @@ struct rdma_cm_id { uint8_t port_num; }; -struct rdma_multicast_data { - void *context; - struct sockaddr addr; - uint8_t pad[sizeof(struct sockaddr_in6) - - sizeof(struct sockaddr)]; +struct rdma_conn_param { + const void *private_data; + uint8_t private_data_len; + uint8_t responder_resources; + uint8_t initiator_depth; + uint8_t flow_control; + uint8_t retry_count; /* ignored when accepting */ + uint8_t rnr_retry_count; + /* Fields below ignored if a QP is created on the rdma_cm_id. */ + uint8_t srq; + uint32_t qp_num; +}; + +struct rdma_ud_param { + const void *private_data; + uint8_t private_data_len; + struct ibv_ah_attr ah_attr; + uint32_t qp_num; + uint32_t qkey; }; struct rdma_cm_event { @@ -117,8 +133,10 @@ struct rdma_cm_event { struct rdma_cm_id *listen_id; enum rdma_cm_event_type event; int status; - void *private_data; - uint8_t private_data_len; + union { + struct rdma_conn_param conn; + struct rdma_ud_param ud; + } param; }; /** @@ -210,20 +228,6 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd, */ void rdma_destroy_qp(struct rdma_cm_id *id); -struct rdma_conn_param { - const void *private_data; - uint8_t private_data_len; - uint8_t responder_resources; - uint8_t initiator_depth; - uint8_t flow_control; - uint8_t retry_count; /* ignored when accepting */ - uint8_t rnr_retry_count; - /* Fields below ignored if a QP is created on the rdma_cm_id. */ - uint8_t srq; - uint32_t qp_num; - enum ibv_qp_type qp_type; -}; - /** * rdma_connect - Initiate an active connection request. * @@ -255,28 +259,25 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data, uint8_t private_data_len); /** - * rdma_disconnect - This function disconnects the associated QP and - * transitions it into the error state. - */ -int rdma_disconnect(struct rdma_cm_id *id); - -/** - * rdma_join_multicast - Join the multicast group specified by the given - * address. - * @id: Communication identifier associated with the request. - * @addr: Multicast address identifying the group to join. - * @context: User-defined context associated with the join request. The - * context is returned to the user through the private_data field in - * the rdma_cm_event. + * rdma_notify - Notifies the RDMA CM of an asynchronous event that has + * occurred on the connection. + * @id: Connection identifier to transition to established. + * @event: Asynchronous event. + * + * This routine should be invoked by users to notify the CM of relevant + * communication events. Events that should be reported to the CM and + * when to report them are: + * + * IB_EVENT_COMM_EST - Used when a message is received on a connected + * QP before an RTU has been received. */ -int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, - void *context); +int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event); /** - * rdma_leave_multicast - Leave the multicast group specified by the given - * address. + * rdma_disconnect - This function disconnects the associated QP and + * transitions it into the error state. */ -int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr); +int rdma_disconnect(struct rdma_cm_id *id); /** * rdma_get_cm_event - Retrieves the next pending communications event, @@ -302,40 +303,17 @@ int rdma_get_cm_event(struct rdma_event_channel *channel, */ int rdma_ack_cm_event(struct rdma_cm_event *event); -/** - * rdma_get_option - Retrieve options for an rdma_cm_id. - * @id: Communication identifier to retrieve option for. - * @level: Protocol level of the option to retrieve. - * @optname: Name of the option to retrieve. - * @optval: Buffer to receive the returned options. - * @optlen: On input, the size of the %optval buffer. On output, the - * size of the returned data. - */ -int rdma_get_option(struct rdma_cm_id *id, int level, int optname, - void *optval, size_t *optlen); - -/** - * rdma_set_option - Set options for an rdma_cm_id. - * @id: Communication identifier to set option for. - * @level: Protocol level of the option to set. - * @optname: Name of the option to set. - * @optval: Reference to the option data. - * @optlen: The size of the %optval buffer. - */ -int rdma_set_option(struct rdma_cm_id *id, int level, int optname, - void *optval, size_t optlen); - static inline uint16_t rdma_get_src_port(struct rdma_cm_id *id) { - return id->route.addr.src_addr.sin6_family == PF_INET6 ? - id->route.addr.src_addr.sin6_port : + return id->route.addr.src_addr.sa_family == PF_INET6 ? + ((struct sockaddr_in6 *) &id->route.addr.src_addr)->sin6_port : ((struct sockaddr_in *) &id->route.addr.src_addr)->sin_port; } static inline uint16_t rdma_get_dst_port(struct rdma_cm_id *id) { - return id->route.addr.dst_addr.sin6_family == PF_INET6 ? - id->route.addr.dst_addr.sin6_port : + return id->route.addr.dst_addr.sa_family == PF_INET6 ? + ((struct sockaddr_in6 *) &id->route.addr.dst_addr)->sin6_port : ((struct sockaddr_in *) &id->route.addr.dst_addr)->sin_port; } diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h index 45d5c37e..cd2d0aad 100644 --- a/include/rdma/rdma_cma_abi.h +++ b/include/rdma/rdma_cma_abi.h @@ -33,14 +33,15 @@ #ifndef RDMA_CMA_ABI_H #define RDMA_CMA_ABI_H +#include #include /* * This file must be kept in sync with the kernel's version of rdma_user_cm.h */ -#define RDMA_USER_CM_MIN_ABI_VERSION 1 -#define RDMA_USER_CM_MAX_ABI_VERSION 2 +#define RDMA_USER_CM_MIN_ABI_VERSION 3 +#define RDMA_USER_CM_MAX_ABI_VERSION 3 #define RDMA_MAX_PRIVATE_DATA 256 @@ -60,9 +61,7 @@ enum { UCMA_CMD_GET_EVENT, UCMA_CMD_GET_OPTION, UCMA_CMD_SET_OPTION, - UCMA_CMD_GET_DST_ATTR, - UCMA_CMD_JOIN_MCAST, - UCMA_CMD_LEAVE_MCAST + UCMA_CMD_NOTIFY }; struct ucma_abi_cmd_hdr { @@ -71,11 +70,6 @@ struct ucma_abi_cmd_hdr { __u16 out; }; -struct ucma_abi_create_id_v1 { - __u64 uid; - __u64 response; -}; - struct ucma_abi_create_id { __u64 uid; __u64 response; @@ -133,7 +127,7 @@ struct ucma_abi_query_route_resp { struct ucma_abi_conn_param { __u32 qp_num; - __u32 qp_type; + __u32 reserved; __u8 private_data[RDMA_MAX_PRIVATE_DATA]; __u8 private_data_len; __u8 srq; @@ -145,6 +139,15 @@ struct ucma_abi_conn_param { __u8 valid; }; +struct ucma_abi_ud_param { + __u32 qp_num; + __u32 qkey; + struct ibv_kern_ah_attr ah_attr; + __u8 private_data[RDMA_MAX_PRIVATE_DATA]; + __u8 private_data_len; + __u8 reserved[7]; +}; + struct ucma_abi_connect { struct ucma_abi_conn_param conn_param; __u32 id; @@ -180,27 +183,9 @@ struct ucma_abi_init_qp_attr { __u32 qp_state; }; -struct ucma_abi_join_mcast { - __u32 id; - struct sockaddr_in6 addr; - __u64 uid; -}; - -struct ucma_abi_leave_mcast { - __u32 id; - struct sockaddr_in6 addr; -}; - -struct ucma_abi_dst_attr_resp { - __u32 remote_qpn; - __u32 remote_qkey; - struct ibv_kern_ah_attr ah_attr; -}; - -struct ucma_abi_get_dst_attr { - __u64 response; - struct sockaddr_in6 addr; +struct ucma_abi_notify { __u32 id; + __u32 event; }; struct ucma_abi_get_event { @@ -212,30 +197,10 @@ struct ucma_abi_event_resp { __u32 id; __u32 event; __u32 status; - __u8 private_data_len; - __u8 reserved[3]; - __u8 private_data[RDMA_MAX_PRIVATE_DATA]; -}; - -struct ucma_abi_get_option { - __u64 response; - __u64 optval; - __u32 id; - __u32 level; - __u32 optname; - __u32 optlen; -}; - -struct ucma_abi_get_option_resp { - __u32 optlen; -}; - -struct ucma_abi_set_option { - __u64 optval; - __u32 id; - __u32 level; - __u32 optname; - __u32 optlen; + union { + struct ucma_abi_conn_param conn; + struct ucma_abi_ud_param ud; + } param; }; #endif /* RDMA_CMA_ABI_H */ diff --git a/include/rdma/rdma_cma_ib.h b/include/rdma/rdma_cma_ib.h deleted file mode 100644 index 6c33c615..00000000 --- a/include/rdma/rdma_cma_ib.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2006 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - * - */ - -#if !defined(RDMA_CMA_IB_H) -#define RDMA_CMA_IB_H - -#include - - -/* IB specific option names for get/set. */ -enum { - IB_PATH_OPTIONS = 1, /* struct ibv_kern_path_rec */ - IB_CM_REQ_OPTIONS = 2 /* struct ib_cm_req_opt */ -}; - -struct ib_cm_req_opt { - uint8_t remote_cm_response_timeout; - uint8_t local_cm_response_timeout; - uint8_t max_cm_retries; -}; - -/** - * rdma_get_dst_attr - Retrieve information about a UDP destination. - * @id: Connection identifier associated with the request. - * @addr: Address of remote destination to retrieve information about. - * @ah_attr: Address handle attributes. A caller uses these attributes to - * create an address handle when communicating with the destination. - * @qpn: The remote QP number associated with the UDP address. - * @qkey: The QKey of the remote QP. - * - * Users must have called rdma_connect() to resolve the destination information. - */ -int rdma_get_dst_attr(struct rdma_cm_id *id, struct sockaddr *addr, - struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn, - uint32_t *remote_qkey); - -/* - * Global qkey value for all UD QPs and multicast groups created via the - * RDMA CM. - */ -#define RDMA_UD_QKEY 0x01234567 - -#endif /* RDMA_CMA_IB_H */ diff --git a/src/cma.c b/src/cma.c index 3ccfcb8e..043dab88 100644 --- a/src/cma.c +++ b/src/cma.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005 Intel Corporation. All rights reserved. + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -54,7 +54,6 @@ #include #include #include -#include #define PFX "librdmacm: " @@ -118,6 +117,12 @@ struct cma_id_private { uint32_t handle; }; +struct cma_event { + struct rdma_cm_event event; + uint8_t private_data[RDMA_MAX_PRIVATE_DATA]; + struct cma_id_private *id_priv; +}; + static struct cma_device *cma_dev_array; static int cma_dev_cnt; static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER; @@ -335,41 +340,6 @@ err: ucma_free_id(id_priv); return NULL; } -static int ucma_create_id_v1(struct rdma_event_channel *channel, - struct rdma_cm_id **id, void *context, - enum rdma_port_space ps) -{ - struct ucma_abi_create_id_resp *resp; - struct ucma_abi_create_id_v1 *cmd; - struct cma_id_private *id_priv; - void *msg; - int ret, size; - - if (ps != RDMA_PS_TCP) { - fprintf(stderr, "librdmacm: Kernel ABI does not support " - "requested port space.\n"); - return -EPROTONOSUPPORT; - } - - id_priv = ucma_alloc_id(channel, context, ps); - if (!id_priv) - return -ENOMEM; - - CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size); - cmd->uid = (uintptr_t) id_priv; - - ret = write(channel->fd, msg, size); - if (ret != size) - goto err; - - id_priv->handle = resp->id; - *id = &id_priv->id; - return 0; - -err: ucma_free_id(id_priv); - return ret; -} - int rdma_create_id(struct rdma_event_channel *channel, struct rdma_cm_id **id, void *context, enum rdma_port_space ps) @@ -384,9 +354,6 @@ int rdma_create_id(struct rdma_event_channel *channel, if (ret) return ret; - if (abi_ver == 1) - return ucma_create_id_v1(channel, id, context, ps); - id_priv = ucma_alloc_id(channel, context, ps); if (!id_priv) return -ENOMEM; @@ -492,9 +459,9 @@ static int ucma_query_route(struct rdma_cm_id *id) sizeof id->route.addr.addr.ibaddr.dgid); id->route.addr.addr.ibaddr.pkey = resp->ib_route[0].pkey; memcpy(&id->route.addr.src_addr, &resp->src_addr, - sizeof id->route.addr.src_addr); + sizeof resp->src_addr); memcpy(&id->route.addr.dst_addr, &resp->dst_addr, - sizeof id->route.addr.dst_addr); + sizeof resp->dst_addr); if (!id_priv->cma_dev && resp->node_guid) { ret = ucma_get_device(id_priv, resp->node_guid); @@ -696,7 +663,7 @@ static int ucma_init_ib_qp(struct cma_id_private *id_priv, struct ibv_qp *qp) qp_attr.port_num = id_priv->id.port_num; qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE; + qp_attr.qp_access_flags = 0; return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS | IBV_QP_PKEY_INDEX | IBV_QP_PORT); } @@ -767,11 +734,9 @@ void rdma_destroy_qp(struct rdma_cm_id *id) static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst, struct rdma_conn_param *src, - uint32_t qp_num, - enum ibv_qp_type qp_type, uint8_t srq) + uint32_t qp_num, uint8_t srq) { dst->qp_num = qp_num; - dst->qp_type = qp_type; dst->srq = srq; dst->responder_resources = src->responder_resources; dst->initiator_depth = src->initiator_depth; @@ -799,12 +764,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) cmd->id = id_priv->handle; if (id->qp) ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, - id->qp->qp_num, id->qp->qp_type, + id->qp->qp_num, (id->qp->srq != NULL)); else ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, conn_param->qp_num, - conn_param->qp_type, conn_param->srq); ret = write(id->channel->fd, msg, size); @@ -852,12 +816,11 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) cmd->uid = (uintptr_t) id_priv; if (id->qp) ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, - id->qp->qp_num, id->qp->qp_type, + id->qp->qp_num, (id->qp->srq != NULL)); else ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, conn_param->qp_num, - conn_param->qp_type, conn_param->srq); ret = write(id->channel->fd, msg, size); @@ -894,6 +857,25 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data, return 0; } +int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event) +{ + struct ucma_abi_notify *cmd; + struct cma_id_private *id_priv; + void *msg; + int ret, size; + + CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_NOTIFY, size); + + id_priv = container_of(id, struct cma_id_private, id); + cmd->id = id_priv->handle; + cmd->event = event; + ret = write(id->channel->fd, msg, size); + if (ret != size) + return (ret > 0) ? -ENODATA : ret; + + return 0; +} + int rdma_disconnect(struct rdma_cm_id *id) { struct ucma_abi_disconnect *cmd; @@ -925,80 +907,6 @@ int rdma_disconnect(struct rdma_cm_id *id) return 0; } -int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, - void *context) -{ - struct ucma_abi_join_mcast *cmd; - struct cma_id_private *id_priv; - void *msg; - int ret, size, addrlen; - - addrlen = ucma_addrlen(addr); - if (!addrlen) - return -EINVAL; - - CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_JOIN_MCAST, size); - id_priv = container_of(id, struct cma_id_private, id); - cmd->id = id_priv->handle; - memcpy(&cmd->addr, addr, addrlen); - cmd->uid = (uintptr_t) context; - - ret = write(id->channel->fd, msg, size); - if (ret != size) - return (ret > 0) ? -ENODATA : ret; - - return 0; -} - -int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) -{ - struct ucma_abi_leave_mcast *cmd; - struct cma_id_private *id_priv; - void *msg; - int ret, size, addrlen; - struct ibv_ah_attr ah_attr; - uint32_t qp_info; - - addrlen = ucma_addrlen(addr); - if (!addrlen) - return -EINVAL; - - CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_LEAVE_MCAST, size); - id_priv = container_of(id, struct cma_id_private, id); - cmd->id = id_priv->handle; - memcpy(&cmd->addr, addr, addrlen); - - if (id->qp) { - ret = rdma_get_dst_attr(id, addr, &ah_attr, &qp_info, &qp_info); - if (ret) - goto out; - - ret = ibv_detach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid); - if (ret) - goto out; - } - - ret = write(id->channel->fd, msg, size); - if (ret != size) - ret = (ret > 0) ? -ENODATA : ret; -out: - return ret; -} - -static void ucma_copy_event_from_kern(struct rdma_cm_event *dst, - struct ucma_abi_event_resp *src) -{ - dst->event = src->event; - dst->status = src->status; - dst->private_data_len = src->private_data_len; - if (src->private_data_len) { - dst->private_data = dst + 1; - memcpy(dst->private_data, src->private_data, - src->private_data_len); - } else - dst->private_data = NULL; -} - static void ucma_complete_event(struct cma_id_private *id_priv) { pthread_mutex_lock(&id_priv->mut); @@ -1009,36 +917,34 @@ static void ucma_complete_event(struct cma_id_private *id_priv) int rdma_ack_cm_event(struct rdma_cm_event *event) { - struct rdma_cm_id *id; + struct cma_event *evt; if (!event) return -EINVAL; - id = (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) ? - event->listen_id : event->id; + evt = container_of(event, struct cma_event, event); - ucma_complete_event(container_of(id, struct cma_id_private, id)); - free(event); + ucma_complete_event(evt->id_priv); + free(evt); return 0; } -static int ucma_process_conn_req(struct rdma_cm_event *event, +static int ucma_process_conn_req(struct cma_event *evt, uint32_t handle) { - struct cma_id_private *listen_id_priv, *id_priv; + struct cma_id_private *id_priv; int ret; - listen_id_priv = container_of(event->id, struct cma_id_private, id); - id_priv = ucma_alloc_id(event->id->channel, event->id->context, - event->id->ps); + id_priv = ucma_alloc_id(evt->id_priv->id.channel, + evt->id_priv->id.context, evt->id_priv->id.ps); if (!id_priv) { - ucma_destroy_kern_id(event->id->channel->fd, handle); + ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle); ret = -ENOMEM; goto err; } - event->listen_id = event->id; - event->id = &id_priv->id; + evt->event.listen_id = &evt->id_priv->id; + evt->event.id = &id_priv->id; id_priv->handle = handle; ret = ucma_query_route(&id_priv->id); @@ -1049,7 +955,7 @@ static int ucma_process_conn_req(struct rdma_cm_event *event, return 0; err: - ucma_complete_event(listen_id_priv); + ucma_complete_event(evt->id_priv); return ret; } @@ -1093,34 +999,42 @@ static int ucma_process_establish(struct rdma_cm_id *id) return ret; } -static void ucma_process_mcast(struct rdma_cm_id *id, struct rdma_cm_event *evt) +static void ucma_copy_conn_event(struct cma_event *event, + struct ucma_abi_conn_param *src) { - struct ucma_abi_join_mcast kmc_data; - struct rdma_multicast_data *mc_data; - struct ibv_ah_attr ah_attr; - uint32_t qp_info; + struct rdma_conn_param *dst = &event->event.param.conn; - kmc_data = *(struct ucma_abi_join_mcast *) evt->private_data; + dst->private_data_len = src->private_data_len; + if (src->private_data_len) { + dst->private_data = &event->private_data; + memcpy(&event->private_data, src->private_data, + src->private_data_len); + } - mc_data = evt->private_data; - mc_data->context = (void *) (uintptr_t) kmc_data.uid; - memcpy(&mc_data->addr, &kmc_data.addr, - ucma_addrlen((struct sockaddr *) &kmc_data.addr)); + dst->responder_resources = src->responder_resources; + dst->initiator_depth = src->initiator_depth; + dst->flow_control = src->flow_control; + dst->retry_count = src->retry_count; + dst->rnr_retry_count = src->rnr_retry_count; + dst->srq = src->srq; + dst->qp_num = src->qp_num; +} - if (evt->status || !id->qp) - return; +static void ucma_copy_ud_event(struct cma_event *event, + struct ucma_abi_ud_param *src) +{ + struct rdma_ud_param *dst = &event->event.param.ud; - evt->status = rdma_get_dst_attr(id, &mc_data->addr, &ah_attr, - &qp_info, &qp_info); - if (evt->status) - goto err; + dst->private_data_len = src->private_data_len; + if (src->private_data_len) { + dst->private_data = &event->private_data; + memcpy(&event->private_data, src->private_data, + src->private_data_len); + } - evt->status = ibv_attach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid); - if (evt->status) - goto err; - return; -err: - evt->event = RDMA_CM_EVENT_MULTICAST_ERROR; + ibv_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr); + dst->qp_num = src->qp_num; + dst->qkey = src->qkey; } int rdma_get_cm_event(struct rdma_event_channel *channel, @@ -1128,8 +1042,7 @@ int rdma_get_cm_event(struct rdma_event_channel *channel, { struct ucma_abi_event_resp *resp; struct ucma_abi_get_event *cmd; - struct cma_id_private *id_priv; - struct rdma_cm_event *evt; + struct cma_event *evt; void *msg; int ret, size; @@ -1140,155 +1053,102 @@ int rdma_get_cm_event(struct rdma_event_channel *channel, if (!event) return -EINVAL; - evt = malloc(sizeof *evt + RDMA_MAX_PRIVATE_DATA); + evt = malloc(sizeof *evt); if (!evt) return -ENOMEM; retry: + memset(evt, 0, sizeof *evt); CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size); ret = write(channel->fd, msg, size); if (ret != size) { free(evt); return (ret > 0) ? -ENODATA : ret; } - - id_priv = (void *) (uintptr_t) resp->uid; - evt->id = &id_priv->id; - ucma_copy_event_from_kern(evt, resp); - switch (evt->event) { + evt->event.event = resp->event; + switch (resp->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: - evt->status = ucma_query_route(&id_priv->id); - if (evt->status) - evt->event = RDMA_CM_EVENT_ADDR_ERROR; + evt->id_priv = (void *) (uintptr_t) resp->uid; + evt->event.id = &evt->id_priv->id; + evt->event.status = ucma_query_route(&evt->id_priv->id); + if (evt->event.status) + evt->event.event = RDMA_CM_EVENT_ADDR_ERROR; break; case RDMA_CM_EVENT_ROUTE_RESOLVED: - evt->status = ucma_query_route(&id_priv->id); - if (evt->status) - evt->event = RDMA_CM_EVENT_ROUTE_ERROR; + evt->id_priv = (void *) (uintptr_t) resp->uid; + evt->event.id = &evt->id_priv->id; + evt->event.status = ucma_query_route(&evt->id_priv->id); + if (evt->event.status) + evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR; break; case RDMA_CM_EVENT_CONNECT_REQUEST: + evt->id_priv = (void *) (uintptr_t) resp->uid; + if (evt->id_priv->id.ps == RDMA_PS_TCP) + ucma_copy_conn_event(evt, &resp->param.conn); + else + ucma_copy_ud_event(evt, &resp->param.ud); + ret = ucma_process_conn_req(evt, resp->id); if (ret) goto retry; break; case RDMA_CM_EVENT_CONNECT_RESPONSE: - evt->status = ucma_process_conn_resp(id_priv); - if (!evt->status) - evt->event = RDMA_CM_EVENT_ESTABLISHED; + evt->id_priv = (void *) (uintptr_t) resp->uid; + evt->event.id = &evt->id_priv->id; + ucma_copy_conn_event(evt, &resp->param.conn); + evt->event.status = ucma_process_conn_resp(evt->id_priv); + if (!evt->event.status) + evt->event.event = RDMA_CM_EVENT_ESTABLISHED; else { - evt->event = RDMA_CM_EVENT_CONNECT_ERROR; - id_priv->connect_error = 1; + evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR; + evt->id_priv->connect_error = 1; } break; case RDMA_CM_EVENT_ESTABLISHED: - if (id_priv->id.ps == RDMA_PS_UDP) + evt->id_priv = (void *) (uintptr_t) resp->uid; + evt->event.id = &evt->id_priv->id; + if (evt->id_priv->id.ps == RDMA_PS_UDP) { + ucma_copy_ud_event(evt, &resp->param.ud); break; + } - evt->status = ucma_process_establish(&id_priv->id); - if (evt->status) { - evt->event = RDMA_CM_EVENT_CONNECT_ERROR; - id_priv->connect_error = 1; + ucma_copy_conn_event(evt, &resp->param.conn); + evt->event.status = ucma_process_establish(&evt->id_priv->id); + if (evt->event.status) { + evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR; + evt->id_priv->connect_error = 1; } break; case RDMA_CM_EVENT_REJECTED: - if (id_priv->connect_error) { - ucma_complete_event(id_priv); + evt->id_priv = (void *) (uintptr_t) resp->uid; + if (evt->id_priv->connect_error) { + ucma_complete_event(evt->id_priv); goto retry; } - ucma_modify_qp_err(evt->id); + evt->event.id = &evt->id_priv->id; + ucma_copy_conn_event(evt, &resp->param.conn); + ucma_modify_qp_err(evt->event.id); break; case RDMA_CM_EVENT_DISCONNECTED: - if (id_priv->connect_error) { - ucma_complete_event(id_priv); + evt->id_priv = (void *) (uintptr_t) resp->uid; + if (evt->id_priv->connect_error) { + ucma_complete_event(evt->id_priv); goto retry; } - break; - case RDMA_CM_EVENT_MULTICAST_JOIN: - case RDMA_CM_EVENT_MULTICAST_ERROR: - ucma_process_mcast(&id_priv->id, evt); + evt->event.id = &evt->id_priv->id; + ucma_copy_conn_event(evt, &resp->param.conn); break; default: + evt->id_priv = (void *) (uintptr_t) resp->uid; + evt->event.id = &evt->id_priv->id; + if (evt->id_priv->id.ps == RDMA_PS_TCP) + ucma_copy_conn_event(evt, &resp->param.conn); + else + ucma_copy_ud_event(evt, &resp->param.ud); break; } - *event = evt; - return 0; -} - -int rdma_get_option(struct rdma_cm_id *id, int level, int optname, - void *optval, size_t *optlen) -{ - struct ucma_abi_get_option_resp *resp; - struct ucma_abi_get_option *cmd; - struct cma_id_private *id_priv; - void *msg; - int ret, size; - - CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_OPTION, size); - id_priv = container_of(id, struct cma_id_private, id); - cmd->id = id_priv->handle; - cmd->optval = (uintptr_t) optval; - cmd->level = level; - cmd->optname = optname; - cmd->optlen = *optlen; - - ret = write(id->channel->fd, msg, size); - if (ret != size) - return (ret > 0) ? -ENODATA : ret; - - *optlen = resp->optlen; - return 0; -} - -int rdma_set_option(struct rdma_cm_id *id, int level, int optname, - void *optval, size_t optlen) -{ - struct ucma_abi_set_option *cmd; - struct cma_id_private *id_priv; - void *msg; - int ret, size; - - CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_SET_OPTION, size); - id_priv = container_of(id, struct cma_id_private, id); - cmd->id = id_priv->handle; - cmd->optval = (uintptr_t) optval; - cmd->level = level; - cmd->optname = optname; - cmd->optlen = optlen; - - ret = write(id->channel->fd, msg, size); - if (ret != size) - return (ret > 0) ? -ENODATA : ret; - - return 0; -} - -int rdma_get_dst_attr(struct rdma_cm_id *id, struct sockaddr *addr, - struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn, - uint32_t *remote_qkey) -{ - struct ucma_abi_dst_attr_resp *resp; - struct ucma_abi_get_dst_attr *cmd; - struct cma_id_private *id_priv; - void *msg; - int ret, size, addrlen; - - addrlen = ucma_addrlen(addr); - if (!addrlen) - return -EINVAL; - - CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_DST_ATTR, size); - id_priv = container_of(id, struct cma_id_private, id); - cmd->id = id_priv->handle; - memcpy(&cmd->addr, addr, addrlen); - - ret = write(id->channel->fd, msg, size); - if (ret != size) - return (ret > 0) ? -ENODATA : ret; - - ibv_copy_ah_attr_from_kern(ah_attr, &resp->ah_attr); - *remote_qpn = resp->remote_qpn; - *remote_qkey = resp->remote_qkey; + *event = &evt->event; return 0; } diff --git a/src/librdmacm.map b/src/librdmacm.map index 52986ca6..ed3f35a3 100644 --- a/src/librdmacm.map +++ b/src/librdmacm.map @@ -1,4 +1,4 @@ -RDMACM_1.0 { +RDMACM_2.0 { global: rdma_create_event_channel; rdma_destroy_event_channel; @@ -19,8 +19,6 @@ RDMACM_1.0 { rdma_get_option; rdma_set_option; rdma_get_dst_attr; - rdma_join_multicast; - rdma_leave_multicast; rdma_get_devices; rdma_free_devices; local: *; -- 2.41.0