From 04cf886f5d20ce6ce7f662c2058a858ab560b29e Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Fri, 5 Apr 2013 21:59:40 -0700 Subject: [PATCH] Refresh of af_ib --- src/acm.c | 43 +++++++++++++++++++--------------------- src/cma.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 23 deletions(-) diff --git a/src/acm.c b/src/acm.c index ce077401..a96e49f1 100755 --- a/src/acm.c +++ b/src/acm.c @@ -129,41 +129,38 @@ void ucma_ib_cleanup(void) static int ucma_ib_set_addr(struct rdma_addrinfo *ib_rai, struct rdma_addrinfo *rai) { - struct sockaddr/*_ib*/ *src, *dst; -// struct ibv_path_record *path; + struct sockaddr_ib *src, *dst; + struct ibv_path_record *path; - src = calloc(1, rai->ai_src_len);//sizeof *src); + src = calloc(1, sizeof *src); if (!src) return ERR(ENOMEM); - dst = calloc(1, rai->ai_dst_len);//sizeof *dst); + dst = calloc(1, sizeof *dst); if (!dst) { free(src); return ERR(ENOMEM); } - memcpy(src, rai->ai_src_addr, rai->ai_src_len); - memcpy(dst, rai->ai_dst_addr, rai->ai_dst_len); - ((struct sockaddr_in *) dst)->sin_addr.s_addr = htonl(0x7f000001); -// path = &((struct ibv_path_data *) ib_rai->ai_route)->path; + path = &((struct ibv_path_data *) ib_rai->ai_route)->path; -// src->sib_family = AF_IB; -// src->sib_pkey = path->pkey; -// src->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8); -// memcpy(&src->sib_addr, &path->sgid, 16); -// ucma_set_sid(ib_rai->ai_port_space, rai->ai_src_addr, src); + src->sib_family = AF_IB; + src->sib_pkey = path->pkey; + src->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8); + memcpy(&src->sib_addr, &path->sgid, 16); + ucma_set_sid(ib_rai->ai_port_space, rai->ai_src_addr, src); -// dst->sib_family = AF_IB; -// dst->sib_pkey = path->pkey; -// dst->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8); -// memcpy(&dst->sib_addr, &path->dgid, 16); -// ucma_set_sid(ib_rai->ai_port_space, rai->ai_dst_addr, dst); + dst->sib_family = AF_IB; + dst->sib_pkey = path->pkey; + dst->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8); + memcpy(&dst->sib_addr, &path->dgid, 16); + ucma_set_sid(ib_rai->ai_port_space, rai->ai_dst_addr, dst); ib_rai->ai_src_addr = (struct sockaddr *) src; - ib_rai->ai_src_len = rai->ai_src_len;//sizeof(*src); + ib_rai->ai_src_len = sizeof(*src); ib_rai->ai_dst_addr = (struct sockaddr *) dst; - ib_rai->ai_dst_len = rai->ai_dst_len;//sizeof(*dst); + ib_rai->ai_dst_len = sizeof(*dst); return 0; } @@ -231,8 +228,8 @@ static void ucma_resolve_af_ib(struct rdma_addrinfo **rai) goto err; } -// if (ucma_ib_set_connect(ib_rai, *rai)) -// goto err; + if (ucma_ib_set_connect(ib_rai, *rai)) + goto err; if (ucma_ib_set_addr(ib_rai, *rai)) goto err; @@ -401,7 +398,7 @@ void ucma_ib_resolve(struct rdma_addrinfo **rai, struct rdma_addrinfo *hints) ucma_ib_save_resp(*rai, &msg); - if (/*af_ib_support &&*/ !(hints->ai_flags & RAI_ROUTEONLY) && (*rai)->ai_route_len) + if (af_ib_support && !(hints->ai_flags & RAI_ROUTEONLY) && (*rai)->ai_route_len) ucma_resolve_af_ib(rai); } diff --git a/src/cma.c b/src/cma.c index 2fb9913a..2b3cd4d2 100755 --- a/src/cma.c +++ b/src/cma.c @@ -49,6 +49,7 @@ #include #include #include +#include #include "cma.h" #include "indexer.h" @@ -128,6 +129,13 @@ int af_ib_support; static struct index_map ucma_idm; static fastlock_t idm_lock; +static char *ucma_addr_str(struct sockaddr *addr) +{ + static char buf[32]; + + return inet_ntop(addr->sa_family, &((struct sockaddr_in *) addr)->sin_addr, buf, sizeof buf); +} + static void ucma_cleanup(void) { ucma_ib_cleanup(); @@ -199,6 +207,7 @@ static void ucma_set_af_ib_support(void) ret = rdma_bind_addr(id, (struct sockaddr *) &sib); af_ib_support = !ret; +printf("af_ib support %d", af_ib_support); rdma_destroy_id(id); } @@ -835,6 +844,37 @@ static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr, return ucma_complete(id); } +static int ucma_get_ib_route(struct cma_id_private *id_priv, + struct sockaddr *src_addr, struct sockaddr *dst_addr) +{ + struct rdma_addrinfo hint, *rai; + int ret; + + memset(&hint, 0, sizeof hint); + hint.ai_flags = RAI_ROUTEONLY; + hint.ai_family = dst_addr->sa_family; + hint.ai_src_len = ucma_addrlen(src_addr); + hint.ai_src_addr = src_addr; + hint.ai_dst_len = ucma_addrlen(dst_addr); + hint.ai_dst_addr = dst_addr; + + ret = rdma_getaddrinfo(NULL, NULL, &hint, &rai); + if (ret) + return ret; + + if (rai->ai_route_len) { + id_priv->connect = rai->ai_route; + id_priv->connect_len = rai->ai_route_len; + rai->ai_route = NULL; + rai->ai_route_len = 0; + } else { + ret = -1; + } + + rdma_freeaddrinfo(rai); + return ret; +} + int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, struct sockaddr *dst_addr, int timeout_ms) { @@ -842,6 +882,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, struct cma_id_private *id_priv; int ret, dst_len, src_len; +printf("rdma_resolve_addr init dest %s\n", ucma_addr_str(dst_addr)); dst_len = ucma_addrlen(dst_addr); if (!dst_len) return ERR(EINVAL); @@ -862,6 +903,14 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, memcpy(&cmd.dst_addr, dst_addr, dst_len); cmd.timeout_ms = timeout_ms; + ret = ucma_get_ib_route(id_priv, src_addr, dst_addr); + if (!ret) { + ((struct sockaddr_in *) &cmd.dst_addr)->sin_addr.s_addr = htonl(0x7f000001); + printf("rdma_resolve_addr dest %s\n", ucma_addr_str(&cmd.dst_addr)); + } + +if (src_addr) + printf("rdma_resolve_addr src %s\n", ucma_addr_str(src_addr)); ret = write(id->channel->fd, &cmd, sizeof cmd); if (ret != sizeof cmd) return (ret >= 0) ? ERR(ENODATA) : -1; @@ -874,6 +923,16 @@ static int ucma_set_ib_route(struct rdma_cm_id *id) { struct rdma_addrinfo hint, *rai; int ret; + struct cma_id_private *id_priv; + + id_priv = container_of(id, struct cma_id_private, id); + if (id_priv->connect_len) { + ret = rdma_set_option(id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH, + id_priv->connect, id_priv->connect_len); + free(id_priv->connect); + id_priv->connect_len = 0; + return ret; + } memset(&hint, 0, sizeof hint); hint.ai_flags = RAI_ROUTEONLY; -- 2.41.0