Refresh of af_ib

diff --git a/src/acm.c b/src/acm.c
index ce077401710f21c7b7f9934a4d851f9142926718..a96e49f119ad2e415499108d5b7a2f98372f308d 100755 (executable)
--- a/src/acm.c
+++ b/src/acm.c
@@ -129,41 +129,38 @@ void ucma_ib_cleanup(void)
 static int ucma_ib_set_addr(struct rdma_addrinfo *ib_rai,
                            struct rdma_addrinfo *rai)
 {
-       struct sockaddr/*_ib*/ *src, *dst;
-//     struct ibv_path_record *path;
+       struct sockaddr_ib *src, *dst;
+       struct ibv_path_record *path;
 
-       src = calloc(1, rai->ai_src_len);//sizeof *src);
+       src = calloc(1, sizeof *src);
        if (!src)
                return ERR(ENOMEM);
 
-       dst = calloc(1, rai->ai_dst_len);//sizeof *dst);
+       dst = calloc(1, sizeof *dst);
        if (!dst) {
                free(src);
                return ERR(ENOMEM);
        }
 
-       memcpy(src, rai->ai_src_addr, rai->ai_src_len);
-       memcpy(dst, rai->ai_dst_addr, rai->ai_dst_len);
-       ((struct sockaddr_in *) dst)->sin_addr.s_addr = htonl(0x7f000001);
-//     path = &((struct ibv_path_data *) ib_rai->ai_route)->path;
+       path = &((struct ibv_path_data *) ib_rai->ai_route)->path;
 
-//     src->sib_family = AF_IB;
-//     src->sib_pkey = path->pkey;
-//     src->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
-//     memcpy(&src->sib_addr, &path->sgid, 16);
-//     ucma_set_sid(ib_rai->ai_port_space, rai->ai_src_addr, src);
+       src->sib_family = AF_IB;
+       src->sib_pkey = path->pkey;
+       src->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
+       memcpy(&src->sib_addr, &path->sgid, 16);
+       ucma_set_sid(ib_rai->ai_port_space, rai->ai_src_addr, src);
 
-//     dst->sib_family = AF_IB;
-//     dst->sib_pkey = path->pkey;
-//     dst->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
-//     memcpy(&dst->sib_addr, &path->dgid, 16);
-//     ucma_set_sid(ib_rai->ai_port_space, rai->ai_dst_addr, dst);
+       dst->sib_family = AF_IB;
+       dst->sib_pkey = path->pkey;
+       dst->sib_flowinfo = htonl(ntohl(path->flowlabel_hoplimit) >> 8);
+       memcpy(&dst->sib_addr, &path->dgid, 16);
+       ucma_set_sid(ib_rai->ai_port_space, rai->ai_dst_addr, dst);
 
        ib_rai->ai_src_addr = (struct sockaddr *) src;
-       ib_rai->ai_src_len = rai->ai_src_len;//sizeof(*src);
+       ib_rai->ai_src_len = sizeof(*src);
 
        ib_rai->ai_dst_addr = (struct sockaddr *) dst;
-       ib_rai->ai_dst_len = rai->ai_dst_len;//sizeof(*dst);
+       ib_rai->ai_dst_len = sizeof(*dst);
 
        return 0;
 }
@@ -231,8 +228,8 @@ static void ucma_resolve_af_ib(struct rdma_addrinfo **rai)
                        goto err;
        }
 
-//     if (ucma_ib_set_connect(ib_rai, *rai))
-//             goto err;
+       if (ucma_ib_set_connect(ib_rai, *rai))
+               goto err;
 
        if (ucma_ib_set_addr(ib_rai, *rai))
                goto err;
@@ -401,7 +398,7 @@ void ucma_ib_resolve(struct rdma_addrinfo **rai, struct rdma_addrinfo *hints)
 
        ucma_ib_save_resp(*rai, &msg);
 
-       if (/*af_ib_support &&*/ !(hints->ai_flags & RAI_ROUTEONLY) && (*rai)->ai_route_len)
+       if (af_ib_support && !(hints->ai_flags & RAI_ROUTEONLY) && (*rai)->ai_route_len)
                ucma_resolve_af_ib(rai);
 }
 

diff --git a/src/cma.c b/src/cma.c
index 2fb9913ae6c69c29c74a27ff32955ca5fa7f3eea..2b3cd4d243fb0acd51bad81a543f3bff8145b7f1 100755 (executable)
--- a/src/cma.c
+++ b/src/cma.c
@@ -49,6 +49,7 @@
 #include <stddef.h>
 #include <netdb.h>
 #include <syslog.h>
+#include <arpa/inet.h>
 
 #include "cma.h"
 #include "indexer.h"
@@ -128,6 +129,13 @@ int af_ib_support;
 static struct index_map ucma_idm;
 static fastlock_t idm_lock;
 
+static char *ucma_addr_str(struct sockaddr *addr)
+{
+       static char buf[32];
+
+       return inet_ntop(addr->sa_family, &((struct sockaddr_in *) addr)->sin_addr, buf, sizeof buf);
+}
+
 static void ucma_cleanup(void)
 {
        ucma_ib_cleanup();
@@ -199,6 +207,7 @@ static void ucma_set_af_ib_support(void)
        ret = rdma_bind_addr(id, (struct sockaddr *) &sib);
        af_ib_support = !ret;
 
+printf("af_ib support %d", af_ib_support);
        rdma_destroy_id(id);
 }
 
@@ -835,6 +844,37 @@ static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
        return ucma_complete(id);
 }
 
+static int ucma_get_ib_route(struct cma_id_private *id_priv,
+                            struct sockaddr *src_addr, struct sockaddr *dst_addr)
+{
+       struct rdma_addrinfo hint, *rai;
+       int ret;
+
+       memset(&hint, 0, sizeof hint);
+       hint.ai_flags = RAI_ROUTEONLY;
+       hint.ai_family = dst_addr->sa_family;
+       hint.ai_src_len = ucma_addrlen(src_addr);
+       hint.ai_src_addr = src_addr;
+       hint.ai_dst_len = ucma_addrlen(dst_addr);
+       hint.ai_dst_addr = dst_addr;
+
+       ret = rdma_getaddrinfo(NULL, NULL, &hint, &rai);
+       if (ret)
+               return ret;
+
+       if (rai->ai_route_len) {
+               id_priv->connect = rai->ai_route;
+               id_priv->connect_len = rai->ai_route_len;
+               rai->ai_route = NULL;
+               rai->ai_route_len = 0;
+       } else {
+               ret = -1;
+       }
+
+       rdma_freeaddrinfo(rai);
+       return ret;
+}
+
 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
                      struct sockaddr *dst_addr, int timeout_ms)
 {
@@ -842,6 +882,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
        struct cma_id_private *id_priv;
        int ret, dst_len, src_len;
        
+printf("rdma_resolve_addr init dest %s\n", ucma_addr_str(dst_addr));
        dst_len = ucma_addrlen(dst_addr);
        if (!dst_len)
                return ERR(EINVAL);
@@ -862,6 +903,14 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
        memcpy(&cmd.dst_addr, dst_addr, dst_len);
        cmd.timeout_ms = timeout_ms;
 
+       ret = ucma_get_ib_route(id_priv, src_addr, dst_addr);
+       if (!ret) {
+               ((struct sockaddr_in *) &cmd.dst_addr)->sin_addr.s_addr = htonl(0x7f000001);
+               printf("rdma_resolve_addr dest %s\n", ucma_addr_str(&cmd.dst_addr));
+       }
+
+if (src_addr)
+   printf("rdma_resolve_addr src %s\n", ucma_addr_str(src_addr));
        ret = write(id->channel->fd, &cmd, sizeof cmd);
        if (ret != sizeof cmd)
                return (ret >= 0) ? ERR(ENODATA) : -1;
@@ -874,6 +923,16 @@ static int ucma_set_ib_route(struct rdma_cm_id *id)
 {
        struct rdma_addrinfo hint, *rai;
        int ret;
+       struct cma_id_private *id_priv;
+
+       id_priv = container_of(id, struct cma_id_private, id);
+       if (id_priv->connect_len) {
+               ret = rdma_set_option(id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH,
+                                     id_priv->connect, id_priv->connect_len);
+               free(id_priv->connect);
+               id_priv->connect_len = 0;
+               return ret;
+       }
 
        memset(&hint, 0, sizeof hint);
        hint.ai_flags = RAI_ROUTEONLY;