From ca353a3f985135504c429f82bf5a342ec26d11d4 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Thu, 18 Jul 2013 13:26:15 -0700 Subject: [PATCH] rsockets: Support native IB addressing on connected rsockets Update rsockets to support AF_IB addresses on connected rsockets. Support for datagram rsockets is more difficult as a result of using real UDP sockets for QP resolution, so that support is deferred. For connected sockets, we need to update internal checks to handle AF_IB. Signed-off-by: Sean Hefty --- src/acm.c | 12 ------------ src/cma.h | 12 ++++++++++++ src/rsocket.c | 39 +++++++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/acm.c b/src/acm.c index 33379cfa..45bcb813 100644 --- a/src/acm.c +++ b/src/acm.c @@ -116,18 +116,6 @@ static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER; static int sock = -1; static uint16_t server_port; -struct ib_connect_hdr { - uint8_t cma_version; - uint8_t ip_version; /* IP version: 7:4 */ - uint16_t port; - uint32_t src_addr[4]; - uint32_t dst_addr[4]; -#define cma_src_ip4 src_addr[3] -#define cma_src_ip6 src_addr[0] -#define cma_dst_ip4 dst_addr[3] -#define cma_dst_ip6 dst_addr[0] -}; - static int ucma_set_server_port(void) { FILE *f; diff --git a/src/cma.h b/src/cma.h index e6fba8b0..e944a9a4 100644 --- a/src/cma.h +++ b/src/cma.h @@ -206,6 +206,18 @@ struct ibv_path_data }; #endif +struct ib_connect_hdr { + uint8_t cma_version; + uint8_t ip_version; /* IP version: 7:4 */ + uint16_t port; + uint32_t src_addr[4]; + uint32_t dst_addr[4]; +#define cma_src_ip4 src_addr[3] +#define cma_src_ip6 src_addr[0] +#define cma_dst_ip4 dst_addr[3] +#define cma_dst_ip6 dst_addr[0] +}; + #ifndef SYSCONFDIR #define SYSCONFDIR "/etc" #endif diff --git a/src/rsocket.c b/src/rsocket.c index ef24dfbf..e2b1cc70 100644 --- a/src/rsocket.c +++ b/src/rsocket.c @@ -187,6 +187,16 @@ struct rs_conn_data { struct rs_sge data_buf; }; +struct rs_conn_private_data { + union { + struct rs_conn_data conn_data; + struct { + struct ib_connect_hdr ib_hdr; + struct rs_conn_data conn_data; + } af_ib; + }; +}; + /* * rsocket states are ordered as passive, connecting, connected, disconnected. */ @@ -965,8 +975,13 @@ static void rs_free(struct rsocket *rs) free(rs); } -static void rs_set_conn_data(struct rsocket *rs, struct rdma_conn_param *param, - struct rs_conn_data *conn) +static size_t rs_conn_data_offset(struct rsocket *rs) +{ + return (rs->cm_id->route.addr.src_addr.sa_family == AF_IB) ? + sizeof(struct ib_connect_hdr) : 0; +} + +static void rs_format_conn_data(struct rsocket *rs, struct rs_conn_data *conn) { conn->version = 1; conn->flags = RS_CONN_FLAG_IOMAP | @@ -982,9 +997,6 @@ static void rs_set_conn_data(struct rsocket *rs, struct rdma_conn_param *param, conn->data_buf.addr = htonll((uintptr_t) rs->rbuf); conn->data_buf.length = htonl(rs->rbuf_size >> 1); conn->data_buf.key = htonl(rs->rmr->rkey); - - param->private_data = conn; - param->private_data_len = sizeof *conn; } static void rs_save_conn_data(struct rsocket *rs, struct rs_conn_data *conn) @@ -1063,7 +1075,7 @@ int rsocket(int domain, int type, int protocol) struct rsocket *rs; int index, ret; - if ((domain != PF_INET && domain != PF_INET6) || + if ((domain != AF_INET && domain != AF_INET6 && domain != AF_IB) || ((type != SOCK_STREAM) && (type != SOCK_DGRAM)) || (type == SOCK_STREAM && protocol && protocol != IPPROTO_TCP) || (type == SOCK_DGRAM && protocol && protocol != IPPROTO_UDP)) @@ -1165,7 +1177,8 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen) if (ret < 0) goto err; - creq = (struct rs_conn_data *) new_rs->cm_id->event->param.conn.private_data; + creq = (struct rs_conn_data *) + (new_rs->cm_id->event->param.conn.private_data + rs_conn_data_offset(rs)); if (creq->version != 1) { ret = ERR(ENOTSUP); goto err; @@ -1180,7 +1193,9 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen) rs_save_conn_data(new_rs, creq); param = new_rs->cm_id->event->param.conn; - rs_set_conn_data(new_rs, ¶m, &cresp); + rs_format_conn_data(new_rs, &cresp); + param.private_data = &cresp; + param.private_data_len = sizeof cresp; ret = rdma_accept(new_rs->cm_id, ¶m); if (!ret) new_rs->state = rs_connect_rdwr; @@ -1201,7 +1216,8 @@ err: static int rs_do_connect(struct rsocket *rs) { struct rdma_conn_param param; - struct rs_conn_data creq, *cresp; + struct rs_conn_private_data cdata; + struct rs_conn_data *creq, *cresp; int to, ret; switch (rs->state) { @@ -1246,7 +1262,10 @@ do_connect: break; memset(¶m, 0, sizeof param); - rs_set_conn_data(rs, ¶m, &creq); + creq = (void *) &cdata + rs_conn_data_offset(rs); + rs_format_conn_data(rs, creq); + param.private_data = (void *) creq - rs_conn_data_offset(rs); + param.private_data_len = sizeof(*creq) + rs_conn_data_offset(rs); param.flow_control = 1; param.retry_count = 7; param.rnr_retry_count = 7; -- 2.41.0