From eb0bb5edd74fe71f3624f275b1547b40aa8a4206 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Fri, 25 May 2012 10:48:47 -0700 Subject: [PATCH] librdmacm: Delay ACM connection until resolving an address Avoid creating a connection to the ACM service when it's not needed. For example, if the user of the librdmacm is a server application, it will not use ACM services. Signed-off-by: Sean Hefty --- src/acm.c | 26 ++++++++++++++++++-------- src/addrinfo.c | 2 +- src/cma.c | 4 +--- src/cma.h | 10 ++++------ src/rsocket.c | 9 ++++----- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/acm.c b/src/acm.c index bcf11dad..d59efcf8 100755 --- a/src/acm.c +++ b/src/acm.c @@ -61,7 +61,7 @@ typedef struct acm_msg cma_acm_msg_t; #endif static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER; -static int sock; +static int sock = -1; static short server_port = 6125; struct ib_connect_hdr { @@ -86,15 +86,20 @@ static void ucma_set_server_port(void) } } -void ucma_ib_init(void) +void ucma_acm_init(void) { struct sockaddr_in addr; + static int init; int ret; + if (init) + return; + + pthread_mutex_lock(&acm_lock); ucma_set_server_port(); sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); if (sock < 0) - return; + goto err1; memset(&addr, 0, sizeof addr); addr.sin_family = AF_INET; @@ -102,18 +107,23 @@ void ucma_ib_init(void) addr.sin_port = htons(server_port); ret = connect(sock, (struct sockaddr *) &addr, sizeof(addr)); if (ret) - goto err; + goto err2; + init = 1; + pthread_mutex_unlock(&acm_lock); return; -err: +err2: close(sock); - sock = 0; + sock = -1; +err1: + init = 1; + pthread_mutex_unlock(&acm_lock); } void ucma_ib_cleanup(void) { - if (sock > 0) { + if (sock >= 0) { shutdown(sock, SHUT_RDWR); close(sock); } @@ -322,7 +332,7 @@ void ucma_ib_resolve(struct rdma_addrinfo **rai, struct rdma_addrinfo *hints) struct acm_ep_addr_data *data; int ret; - if (sock <= 0) + if (sock < 0) return; memset(&msg, 0, sizeof msg); diff --git a/src/addrinfo.c b/src/addrinfo.c index 2da35f01..573fb42c 100755 --- a/src/addrinfo.c +++ b/src/addrinfo.c @@ -231,7 +231,7 @@ int rdma_getaddrinfo(char *node, char *service, } if (!(rai->ai_flags & RAI_PASSIVE)) - ucma_ib_resolve(&rai, hints); + ucma_acm_resolve(&rai, hints); *res = rai; return 0; diff --git a/src/cma.c b/src/cma.c index 753612c5..c0cd407d 100755 --- a/src/cma.c +++ b/src/cma.c @@ -128,7 +128,7 @@ int af_ib_support; static void ucma_cleanup(void) { - ucma_ib_cleanup(); + ucma_acm_cleanup(); if (cma_dev_cnt) { while (cma_dev_cnt--) { @@ -274,8 +274,6 @@ int ucma_init(void) ib += (cma_dev->verbs->device->transport_type == IBV_TRANSPORT_IB); } - if (ib) - ucma_ib_init(); cma_dev_cnt = dev_cnt; ucma_set_af_ib_support(); pthread_mutex_unlock(&mut); diff --git a/src/cma.h b/src/cma.h index 2f5215c1..c673bda7 100644 --- a/src/cma.h +++ b/src/cma.h @@ -119,13 +119,11 @@ extern int af_ib_support; #define RAI_ROUTEONLY 0x01000000 #ifdef USE_IB_ACM -void ucma_ib_init(); -void ucma_ib_cleanup(); -void ucma_ib_resolve(struct rdma_addrinfo **rai, struct rdma_addrinfo *hints); +void ucma_acm_cleanup(); +void ucma_acm_resolve(struct rdma_addrinfo **rai, struct rdma_addrinfo *hints); #else -#define ucma_ib_init() -#define ucma_ib_cleanup() -#define ucma_ib_resolve(x, y) +#define ucma_acm_cleanup() +#define ucma_acm_resolve(x, y) #endif /* Define path record definition if using older version of libibverbs */ diff --git a/src/rsocket.c b/src/rsocket.c index 6bc02f7a..5eec5d9c 100644 --- a/src/rsocket.c +++ b/src/rsocket.c @@ -51,7 +51,6 @@ #include "cma.h" #include "indexer.h" -#define RS_INLINE 64 #define RS_OLAP_START_SIZE 2048 #define RS_MAX_TRANSFER 65536 #define RS_QP_SIZE 384 @@ -109,7 +108,7 @@ struct rs_sge { uint32_t length; }; -#define RS_INLINE_MIN (sizeof(struct rs_sge)) +#define RS_MIN_INLINE (sizeof(struct rs_sge)) #define rs_host_is_net() (1 == htonl(1)) #define RS_CONN_FLAG_NET 1 @@ -223,7 +222,7 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs) rs->ctrl_avail = inherited_rs->ctrl_avail; } else { rs->sbuf_size = rs->rbuf_size = RS_BUF_SIZE; - rs->sq_inline = RS_INLINE; + rs->sq_inline = RS_MIN_INLINE; rs->sq_size = rs->rq_size = RS_QP_SIZE; rs->ctrl_avail = RS_QP_CTRL_SIZE; } @@ -1676,8 +1675,8 @@ int rsetsockopt(int socket, int level, int optname, break; case RDMA_INLINE: rs->sq_inline = min(*(uint32_t *) optval, RS_QP_MAX_SIZE); - if (rs->sq_inline < RS_INLINE_MIN) - rs->sq_inline = RS_INLINE_MIN; + if (rs->sq_inline < RS_MIN_INLINE) + rs->sq_inline = RS_MIN_INLINE; break; default: break; -- 2.45.2